Add validator chain provenance

This commit is contained in:
2026-07-07 21:10:29 +00:00
parent b7ad66f0e0
commit d593bfee0a
20 changed files with 845 additions and 324 deletions

View File

@@ -20,13 +20,14 @@ import (
func productionRegistries() (pipeline.Registries, error) {
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(),
Chunkers: pipeline.NewChunkerRegistry(),
Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(),
Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
Inputs: pipeline.NewInputAdapterRegistry(),
Chunkers: pipeline.NewChunkerRegistry(),
Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(),
Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(),
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
if err := seriatim.Register(registries.Inputs); err != nil {
return pipeline.Registries{}, fmt.Errorf("register seriatim input: %w", err)
@@ -93,25 +94,27 @@ func effectiveRegistries(opts Options) (pipeline.Registries, error) {
func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalog {
return pipeline.ModuleCatalog{
Inputs: registries.Inputs,
Chunkers: registries.Chunkers,
Extractors: registries.Extractors,
Mergers: registries.Mergers,
Normalizers: registries.Normalizers,
Validators: registries.Validators,
Outputs: registries.Outputs,
Inputs: registries.Inputs,
Chunkers: registries.Chunkers,
Extractors: registries.Extractors,
Mergers: registries.Mergers,
Normalizers: registries.Normalizers,
Validators: registries.Validators,
ValidatorChains: registries.ValidatorChains,
Outputs: registries.Outputs,
}
}
func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
return pipeline.Registries{
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Validators: catalog.Validators,
Outputs: catalog.Outputs,
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Validators: catalog.Validators,
ValidatorChains: catalog.ValidatorChains,
Outputs: catalog.Outputs,
}
}
@@ -122,6 +125,7 @@ func isEmptyCatalog(catalog pipeline.ModuleCatalog) bool {
catalog.Mergers == nil &&
catalog.Normalizers == nil &&
catalog.Validators == nil &&
catalog.ValidatorChains == nil &&
catalog.Outputs == nil
}
@@ -132,6 +136,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
registries.Mergers == nil &&
registries.Normalizers == nil &&
registries.Validators == nil &&
registries.ValidatorChains == nil &&
registries.Outputs == nil
}

View File

@@ -2931,13 +2931,14 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
mustRegisterOutput(t, outputs, specs["json"])
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Validators: validators,
Outputs: outputs,
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Validators: validators,
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
}
}

View File

@@ -38,10 +38,21 @@ type ArtifactLaneManifest struct {
Extractor string `json:"extractor"`
Merger string `json:"merger"`
Normalizer string `json:"normalizer"`
Validators []string `json:"validators,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ValidatorChainManifest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Validators []ValidatorManifest `json:"validators"`
}
type ValidatorManifest struct {
Key string `json:"key"`
ExecutionClass string `json:"execution_class"`
}
type LLMProfileManifest struct {
ID string `json:"id"`
Provider string `json:"provider,omitempty"`
@@ -100,6 +111,7 @@ type RunManifest struct {
OutputEncoder string `json:"output_encoder,omitempty"`
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
ValidatorChains []ValidatorChainManifest `json:"validator_chains,omitempty"`
References []ReferenceProvenance `json:"references,omitempty"`
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`

View File

@@ -136,12 +136,21 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
Extractor: "event-extractor",
Merger: "appendorder",
Normalizer: "noop",
Validators: []string{"grounded"},
Metadata: map[string]any{
"extractor": map[string]any{"prompt_id": "test.prompt"},
},
},
},
ValidatorChains: []ValidatorChainManifest{
{
Stage: "extract",
LaneID: "events",
ModuleKey: "event-extractor",
Validators: []ValidatorManifest{
{Key: "grounded", ExecutionClass: "deterministic"},
},
},
},
}
gotJSON, err := json.Marshal(manifest)
@@ -154,7 +163,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
t.Fatalf("json.Unmarshal() error = %v", err)
}
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles")
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "validator_chains", "llm_profiles")
profiles, ok := got["llm_profiles"].([]any)
if !ok {
@@ -180,7 +189,20 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
if !ok {
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
}
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "metadata")
chains, ok := got["validator_chains"].([]any)
if !ok {
t.Fatalf("validator_chains = %#v, want array", got["validator_chains"])
}
if len(chains) != 1 {
t.Fatalf("len(validator_chains) = %d, want 1", len(chains))
}
chain, ok := chains[0].(map[string]any)
if !ok {
t.Fatalf("validator_chains[0] = %#v, want object", chains[0])
}
assertHasKeys(t, chain, "stage", "lane_id", "module_key", "validators")
}
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {

View File

@@ -429,13 +429,14 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
mustRegisterOutput(t, outputs, specs["json"])
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Validators: validators,
Outputs: outputs,
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Validators: validators,
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
}
}
@@ -476,7 +477,8 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry,
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
t.Fatalf("register validator: %v", err)
}
}

View File

@@ -92,12 +92,13 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
}
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Outputs: outputs,
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
}
}

View File

@@ -83,6 +83,18 @@ type ResolvedArtifactLane struct {
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}
type ResolvedValidatorChain struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Validators []ResolvedValidator `json:"validators"`
}
type ResolvedValidator struct {
Binding ModuleBinding `json:"binding"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
}
type ResolvedPipeline struct {
ID string
Digest string
@@ -90,17 +102,19 @@ type ResolvedPipeline struct {
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
Output ModuleBinding
}
type ModuleCatalog struct {
Inputs *InputAdapterRegistry
Chunkers *ChunkerRegistry
Extractors *ExtractorRegistry
Mergers *MergerRegistry
Normalizers *NormalizerRegistry
Validators *ValidatorRegistry
Outputs *OutputEncoderRegistry
Inputs *InputAdapterRegistry
Chunkers *ChunkerRegistry
Extractors *ExtractorRegistry
Mergers *MergerRegistry
Normalizers *NormalizerRegistry
Validators *ValidatorRegistry
ValidatorChains *ValidatorChainRegistry
Outputs *OutputEncoderRegistry
}
func Binding(module string) ModuleBinding {
@@ -172,15 +186,21 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule),
}
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
resolved.ValidatorChains = append(resolved.ValidatorChains, chunkValidatorChain)
outputCapabilities := capabilities.clone()
for _, laneID := range selectedLaneIDs {
laneProfile := lanesByID[laneID]
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
lane, validatorChains, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
resolved.ValidatorChains = append(resolved.ValidatorChains, validatorChains...)
outputCapabilities.addSet(laneCapabilities)
}
@@ -208,7 +228,7 @@ func resolveArtifactLane(
options ResolveOptions,
inherited capabilitySet,
catalog ModuleCatalog,
) (ResolvedArtifactLane, capabilitySet, error) {
) (ResolvedArtifactLane, []ResolvedValidatorChain, capabilitySet, error) {
lane := ResolvedArtifactLane{
ID: laneID,
Extract: resolveBinding(profile.Extract, ""),
@@ -217,17 +237,17 @@ func resolveArtifactLane(
Validators: resolveBindings(profile.Validators, ""),
}
if lane.Extract.Module == "" {
return ResolvedArtifactLane{}, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
return ResolvedArtifactLane{}, nil, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
}
capabilities := inherited.clone()
extractSpec, err := extractorSpec(catalog, lane.Extract.Module)
if err != nil {
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
}
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
}
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
@@ -241,17 +261,17 @@ func resolveArtifactLane(
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
return ResolvedArtifactLane{}, nil, nil, err
}
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
if err != nil {
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
}
if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
}
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
@@ -264,17 +284,17 @@ func resolveArtifactLane(
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
return ResolvedArtifactLane{}, nil, nil, err
}
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
capabilities.add(mergeSpec.Provides...)
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
if err != nil {
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
}
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
}
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
@@ -287,22 +307,105 @@ func resolveArtifactLane(
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
return ResolvedArtifactLane{}, nil, nil, err
}
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
capabilities.add(normalizeSpec.Provides...)
if len(lane.Validators) > 0 {
return ResolvedArtifactLane{}, nil, configuredValidatorsError(pipelineID, laneID)
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
}
return lane, capabilities, nil
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
validatorChains := []ResolvedValidatorChain{extractValidatorChain, mergeValidatorChain, normalizeValidatorChain}
return lane, validatorChains, capabilities, nil
}
func configuredValidatorsError(pipelineID string, laneID string) error {
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID)
}
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
chain := ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
ModuleKey: strings.TrimSpace(module),
}
if chain.ModuleKey == "" {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain %q module key must not be empty", pipelineID, stage)
}
switch stage {
case StageChunk, StageExtract, StageMerge, StageNormalize:
default:
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
}
if catalog.ValidatorChains == nil {
return chain, nil
}
bindings := catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
if len(bindings) == 0 {
return chain, nil
}
if catalog.Validators == nil {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator registry must not be nil for %s validator chain on module %q", pipelineID, stage, chain.ModuleKey)
}
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
for _, validator := range bindings {
spec, ok := catalog.Validators.Spec(validator.Module)
if !ok {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
}
chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(validator),
ExecutionClass: spec.ExecutionClass,
})
}
return chain, nil
}
func cloneResolvedValidatorChains(chains []ResolvedValidatorChain) []ResolvedValidatorChain {
if len(chains) == 0 {
return nil
}
out := make([]ResolvedValidatorChain, len(chains))
for i, chain := range chains {
out[i] = ResolvedValidatorChain{
Stage: chain.Stage,
LaneID: strings.TrimSpace(chain.LaneID),
ModuleKey: strings.TrimSpace(chain.ModuleKey),
Validators: cloneResolvedValidators(chain.Validators),
}
}
return out
}
func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator {
if len(validators) == 0 {
return nil
}
out := make([]ResolvedValidator, len(validators))
for i, validator := range validators {
out[i] = ResolvedValidator{
Binding: cloneModuleBinding(validator.Binding),
ExecutionClass: validator.ExecutionClass,
}
}
return out
}
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
return ResolvedReferenceTarget{
Stage: stage,

View File

@@ -110,6 +110,78 @@ func TestResolvePipelineAppliesDefaults(t *testing.T) {
}
}
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: []ModuleBinding{Binding("grounded")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
resolved, err := ResolvePipeline(PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {Extract: Binding("event-extractor")},
},
}, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if len(resolved.ValidatorChains) != 4 {
t.Fatalf("len(ValidatorChains) = %d, want chunk plus lane extract/merge/normalize", len(resolved.ValidatorChains))
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
if extractChain == nil {
t.Fatal("extract validator chain not found")
}
if len(extractChain.Validators) != 1 {
t.Fatalf("extract validators = %#v, want one validator", extractChain.Validators)
}
if extractChain.Validators[0].Binding.Module != "grounded" {
t.Fatalf("extract validator key = %q, want grounded", extractChain.Validators[0].Binding.Module)
}
if extractChain.Validators[0].ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("extract validator execution class = %q, want deterministic", extractChain.Validators[0].ExecutionClass)
}
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", DefaultChunkModule)
if chunkChain == nil {
t.Fatal("chunk validator chain not found")
}
if len(chunkChain.Validators) != 0 {
t.Fatalf("chunk validators = %#v, want explicit empty chain", chunkChain.Validators)
}
}
func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageNormalize,
Module: DefaultNormalizeModule,
Validators: []ModuleBinding{Binding("missing-validator")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
_, err := ResolvePipeline(PipelineProfile{
ID: "invalid-chain",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {Extract: Binding("event-extractor")},
},
}, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
}
if !strings.Contains(err.Error(), "missing-validator") {
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
}
}
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
profile := multiLaneProfile()
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
@@ -955,6 +1027,15 @@ func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName str
t.Fatalf("binding %q not found in %#v", slotName, bindings)
}
func findResolvedValidatorChain(chains []ResolvedValidatorChain, stage ModuleStage, laneID string, module string) *ResolvedValidatorChain {
for i := range chains {
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
return &chains[i]
}
}
return nil
}
func newProfileCatalog(t *testing.T) ModuleCatalog {
t.Helper()
@@ -994,13 +1075,14 @@ func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) Modul
func emptyProfileCatalog() ModuleCatalog {
return ModuleCatalog{
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(),
Outputs: NewOutputEncoderRegistry(),
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(),
ValidatorChains: NewValidatorChainRegistry(),
Outputs: NewOutputEncoderRegistry(),
}
}
@@ -1043,7 +1125,8 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
t.Fatalf("register normalizer spec %#v: %v", spec, err)
}
case StageValidate:
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
if err := catalog.Validators.RegisterWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err)
}
case StageOutput:

View File

@@ -1,78 +0,0 @@
package pipeline
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type rawValidationKey struct {
stage ModuleStage
module string
}
type RawValidationRegistry struct {
chains map[rawValidationKey][]contracts.Validator
}
func NewRawValidationRegistry() *RawValidationRegistry {
return &RawValidationRegistry{
chains: make(map[rawValidationKey][]contracts.Validator),
}
}
func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.Validator) error {
if r == nil {
return fmt.Errorf("raw validation registry must not be nil")
}
normalizedModule := strings.TrimSpace(module)
if normalizedModule == "" {
return fmt.Errorf("raw validation module key must not be empty")
}
switch stage {
case StageChunk, StageExtract, StageMerge, StageNormalize:
default:
return fmt.Errorf("raw validation stage %q is not supported", stage)
}
if len(validators) == 0 {
return fmt.Errorf("raw validation chain for %q %q must not be empty", stage, normalizedModule)
}
chain := make([]contracts.Validator, 0, len(validators))
for i, validator := range validators {
if validator == nil {
return fmt.Errorf("raw validator %d for %q %q must not be nil", i, stage, normalizedModule)
}
if strings.TrimSpace(validator.Name()) == "" {
return fmt.Errorf("raw validator %d for %q %q must not have an empty name", i, stage, normalizedModule)
}
switch validator.ExecutionClass() {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return fmt.Errorf("raw validator %q for %q %q has unsupported execution class %q", validator.Name(), stage, normalizedModule, validator.ExecutionClass())
}
chain = append(chain, validator)
}
if r.chains == nil {
r.chains = make(map[rawValidationKey][]contracts.Validator)
}
key := rawValidationKey{stage: stage, module: normalizedModule}
if _, exists := r.chains[key]; exists {
return fmt.Errorf("raw validation chain for %q %q is already registered", stage, normalizedModule)
}
r.chains[key] = append([]contracts.Validator(nil), chain...)
return nil
}
func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.Validator {
if r == nil {
return nil
}
chain := r.chains[rawValidationKey{stage: stage, module: strings.TrimSpace(module)}]
if len(chain) == 0 {
return nil
}
return append([]contracts.Validator(nil), chain...)
}

View File

@@ -18,14 +18,14 @@ import (
)
type Registries struct {
Inputs *InputAdapterRegistry
Chunkers *ChunkerRegistry
Extractors *ExtractorRegistry
Mergers *MergerRegistry
Normalizers *NormalizerRegistry
Validators *ValidatorRegistry
RawValidators *RawValidationRegistry
Outputs *OutputEncoderRegistry
Inputs *InputAdapterRegistry
Chunkers *ChunkerRegistry
Extractors *ExtractorRegistry
Mergers *MergerRegistry
Normalizers *NormalizerRegistry
Validators *ValidatorRegistry
ValidatorChains *ValidatorChainRegistry
Outputs *OutputEncoderRegistry
}
type Runner struct {
@@ -127,7 +127,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err != nil {
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
}
rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, input.Metadata, attempt)
rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, input.Metadata, input.Pipeline.ValidatorChains, attempt)
if err != nil || rejection != nil {
return false, rejection, err
}
@@ -240,6 +240,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
schema: extractOutput.Schema,
payload: extractOutput.Payload,
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt,
})
if err != nil || rejection != nil {
@@ -296,6 +297,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
schema: mergeOutput.Schema,
payload: mergeOutput.Payload,
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt,
})
if err != nil || rejection != nil {
@@ -346,6 +348,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
schema: normalizeOutput.Schema,
payload: normalizeOutput.Payload,
metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt,
})
if err != nil || rejection != nil {
@@ -378,6 +381,7 @@ type rawValidationTarget struct {
schema contracts.ResponseSchema
payload contracts.RawPayload
metadata map[string]any
chains []ResolvedValidatorChain
attempt int
}
@@ -428,7 +432,7 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
return false, lastRejection, nil
}
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, metadata map[string]any, attempt int) (*contracts.RejectedOutput, error) {
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, metadata map[string]any, chains []ResolvedValidatorChain, attempt int) (*contracts.RejectedOutput, error) {
for _, chunk := range chunks {
_, rejection, err := r.validateRaw(ctx, rawValidationTarget{
stage: StageChunk,
@@ -443,6 +447,7 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
Metadata: cloneMetadata(chunk.Metadata),
},
metadata: metadata,
chains: chains,
attempt: attempt,
})
if err != nil {
@@ -456,10 +461,16 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
}
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
validators := r.registries.RawValidators.Validators(target.stage, target.moduleKey)
if len(validators) == 0 {
chain := resolvedValidatorChain(target.stage, target.laneID, target.moduleKey, target.chains)
if len(chain.Validators) == 0 {
chain = r.registryValidatorChain(target.stage, target.laneID, target.moduleKey)
}
if len(chain.Validators) == 0 {
return nil, nil, nil
}
if r.registries.Validators == nil {
return nil, nil, fmt.Errorf("validator registry must not be nil")
}
request := contracts.ValidationRequest{
Stage: string(target.stage),
@@ -475,7 +486,11 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
}
var warnings []contracts.Warning
for _, validator := range validators {
for _, validatorBinding := range chain.Validators {
validator, err := r.registries.Validators.Build(validatorBinding.Binding.Module)
if err != nil {
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
}
result, err := validator.Validate(ctx, request)
if err != nil {
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
@@ -507,6 +522,60 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
return warnings, nil, nil
}
func (r *Runner) registryValidatorChain(stage ModuleStage, laneID string, moduleKey string) ResolvedValidatorChain {
chain := ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
ModuleKey: strings.TrimSpace(moduleKey),
}
if r == nil || r.registries.ValidatorChains == nil {
return chain
}
bindings := r.registries.ValidatorChains.Validators(stage, moduleKey)
if len(bindings) == 0 {
return chain
}
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
for _, binding := range bindings {
executionClass := contracts.ExecutionClass("")
if r.registries.Validators != nil {
if spec, ok := r.registries.Validators.Spec(binding.Module); ok {
executionClass = spec.ExecutionClass
}
}
chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(binding),
ExecutionClass: executionClass,
})
}
return chain
}
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
for _, chain := range chains {
if chain.Stage != stage {
continue
}
if chain.ModuleKey != moduleKey {
continue
}
if strings.TrimSpace(chain.LaneID) != strings.TrimSpace(laneID) {
continue
}
return ResolvedValidatorChain{
Stage: chain.Stage,
LaneID: chain.LaneID,
ModuleKey: chain.ModuleKey,
Validators: cloneResolvedValidators(chain.Validators),
}
}
return ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
ModuleKey: strings.TrimSpace(moduleKey),
}
}
func (r *Runner) validateRegistries(pipeline ResolvedPipeline) error {
if r.registries.Inputs == nil {
return fmt.Errorf("input registry must not be nil")
@@ -580,16 +649,17 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
pipeline := input.Pipeline
manifest := artifacts.RunManifest{
PipelineID: pipeline.ID,
PipelineDigest: pipeline.Digest,
InputModule: pipeline.Input.Module,
Chunker: pipeline.Chunk.Module,
OutputEncoder: pipeline.Output.Module,
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
RunID: runID,
StartedAt: timePtr(startedAt),
References: ReferenceProvenance(pipeline),
LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
PipelineID: pipeline.ID,
PipelineDigest: pipeline.Digest,
InputModule: pipeline.Input.Module,
Chunker: pipeline.Chunk.Module,
OutputEncoder: pipeline.Output.Module,
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
ValidatorChains: validatorChainManifests(pipeline.ValidatorChains),
RunID: runID,
StartedAt: timePtr(startedAt),
References: ReferenceProvenance(pipeline),
LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
}
// The runner does not currently maintain a cache or idempotency key. Reference
// digests are recorded in manifest provenance and intentionally kept separate
@@ -607,6 +677,29 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
return manifest
}
func validatorChainManifests(chains []ResolvedValidatorChain) []artifacts.ValidatorChainManifest {
if len(chains) == 0 {
return nil
}
manifests := make([]artifacts.ValidatorChainManifest, 0, len(chains))
for _, chain := range chains {
manifest := artifacts.ValidatorChainManifest{
Stage: string(chain.Stage),
LaneID: chain.LaneID,
ModuleKey: chain.ModuleKey,
Validators: make([]artifacts.ValidatorManifest, 0, len(chain.Validators)),
}
for _, validator := range chain.Validators {
manifest.Validators = append(manifest.Validators, artifacts.ValidatorManifest{
Key: validator.Binding.Module,
ExecutionClass: string(validator.ExecutionClass),
})
}
manifests = append(manifests, manifest)
}
return manifests
}
func failOutput(output RunOutput) RunOutput {
if output.Manifest.PipelineID != "" {
populateRawOutputManifest(&output)

View File

@@ -887,8 +887,9 @@ func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"}
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"}
modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
@@ -909,8 +910,9 @@ func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
@@ -933,8 +935,9 @@ func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-merge", approved: []bool{false}, reason: "bad_merge", message: "merge rejected"}
modules.rawValidators = rawValidationRegistry(t, StageMerge, "merge", validator)
validator := &runnerChainValidator{name: "chain-merge", approved: []bool{false}, reason: "bad_merge", message: "merge rejected"}
modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageMerge, "merge", validator)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
@@ -954,8 +957,9 @@ func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
func TestRunRejectedNormalizePreventsOutputForLane(t *testing.T) {
modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-normalize", approved: []bool{false}, reason: "bad_normalize", message: "normalize rejected"}
modules.rawValidators = rawValidationRegistry(t, StageNormalize, "normalize", validator)
validator := &runnerChainValidator{name: "chain-normalize", approved: []bool{false}, reason: "bad_normalize", message: "normalize rejected"}
modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageNormalize, "normalize", validator)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
@@ -999,8 +1003,9 @@ func TestRunRetriesSameModuleInputAfterFrameworkError(t *testing.T) {
func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false, true, true}, reason: "bad_extract", message: "extract rejected"}
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true, true}, reason: "bad_extract", message: "extract rejected"}
modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].Extract.Retries = 1
@@ -1023,8 +1028,9 @@ func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].Extract.Retries = 1
@@ -1312,8 +1318,11 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
if lane.ID != "alpha" || lane.Extractor != "extract-alpha" || lane.Merger != "merge" || lane.Normalizer != "normalize" {
t.Fatalf("ArtifactLanes[0] = %#v, want lane details", lane)
}
if len(lane.Validators) != 0 {
t.Fatalf("lane validators = %#v, want none", lane.Validators)
if len(manifest.ValidatorChains) != 4 {
t.Fatalf("ValidatorChains = %#v, want four validation points", manifest.ValidatorChains)
}
if manifest.ValidatorChains[0].Stage != string(StageChunk) || manifest.ValidatorChains[0].ModuleKey != "chunk" || len(manifest.ValidatorChains[0].Validators) != 0 {
t.Fatalf("chunk validator chain = %#v, want explicit empty chunk chain", manifest.ValidatorChains[0])
}
}
@@ -1454,6 +1463,12 @@ func resolvedPipeline() ResolvedPipeline {
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
},
},
ValidatorChains: []ResolvedValidatorChain{
{Stage: StageChunk, ModuleKey: "chunk"},
{Stage: StageExtract, LaneID: "alpha", ModuleKey: "extract-alpha"},
{Stage: StageMerge, LaneID: "alpha", ModuleKey: "merge"},
{Stage: StageNormalize, LaneID: "alpha", ModuleKey: "normalize"},
},
Output: Binding("output"),
}
}
@@ -1493,8 +1508,8 @@ type runnerModules struct {
extractors map[string]*runnerExtractor
mergers map[string]*runnerMerger
normalizers map[string]*runnerNormalizer
validators map[string]*runnerValidator
rawValidators *RawValidationRegistry
validators map[string]contracts.Validator
validatorChains *ValidatorChainRegistry
output *runnerOutputEncoder
inputBuildErr error
chunkerBuildErr error
@@ -1513,9 +1528,9 @@ func defaultRunnerModules() *runnerModules {
normalizers: map[string]*runnerNormalizer{
"normalize": {key: "normalize"},
},
validators: map[string]*runnerValidator{
"configured": {name: "configured"},
"second-validator": {name: "second-validator"},
validators: map[string]contracts.Validator{
"configured": &runnerValidator{name: "configured"},
"second-validator": &runnerValidator{name: "second-validator"},
},
output: &runnerOutputEncoder{
key: "output",
@@ -1533,14 +1548,14 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
}
registries := Registries{
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(),
RawValidators: modules.rawValidators,
Outputs: NewOutputEncoderRegistry(),
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(),
ValidatorChains: modules.validatorChains,
Outputs: NewOutputEncoderRegistry(),
}
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
if modules.inputBuildErr != nil {
@@ -1807,7 +1822,7 @@ type runnerValidator struct {
requests []contracts.ValidationRequest
}
type runnerRawValidator struct {
type runnerChainValidator struct {
name string
approved []bool
reason string
@@ -1818,15 +1833,15 @@ type runnerRawValidator struct {
requests []contracts.ValidationRequest
}
func (validator *runnerRawValidator) Name() string {
func (validator *runnerChainValidator) Name() string {
return validator.name
}
func (validator *runnerRawValidator) ExecutionClass() contracts.ExecutionClass {
func (validator *runnerChainValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator *runnerRawValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
func (validator *runnerChainValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
validator.calls++
validator.requests = append(validator.requests, req)
if validator.err != nil {
@@ -2037,12 +2052,16 @@ func assertRunError(t *testing.T, err error, want string) {
}
}
func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *RawValidationRegistry {
func validatorChainRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *ValidatorChainRegistry {
t.Helper()
registry := NewRawValidationRegistry()
if err := registry.Register(stage, module, validators...); err != nil {
t.Fatalf("register raw validators: %v", err)
registry := NewValidatorChainRegistry()
bindings := make([]ModuleBinding, 0, len(validators))
for _, validator := range validators {
bindings = append(bindings, Binding(validator.Name()))
}
if err := registry.Register(ValidatorChainMapping{Stage: stage, Module: module, Validators: bindings}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
return registry
}

View File

@@ -0,0 +1,102 @@
package pipeline
import (
"fmt"
"strings"
)
type validatorChainKey struct {
stage ModuleStage
module string
}
type ValidatorChainMapping struct {
Stage ModuleStage `json:"stage"`
Module string `json:"module"`
Validators []ModuleBinding `json:"validators,omitempty"`
}
type ValidatorChainRegistry struct {
chains map[validatorChainKey][]ModuleBinding
}
func NewValidatorChainRegistry() *ValidatorChainRegistry {
return &ValidatorChainRegistry{
chains: make(map[validatorChainKey][]ModuleBinding),
}
}
func (r *ValidatorChainRegistry) Register(mapping ValidatorChainMapping) error {
if r == nil {
return fmt.Errorf("validator chain registry must not be nil")
}
normalized, err := normalizeValidatorChainMapping(mapping)
if err != nil {
return err
}
if r.chains == nil {
r.chains = make(map[validatorChainKey][]ModuleBinding)
}
key := validatorChainKey{stage: normalized.Stage, module: normalized.Module}
if _, exists := r.chains[key]; exists {
return fmt.Errorf("validator chain for %q %q is already registered", normalized.Stage, normalized.Module)
}
r.chains[key] = cloneModuleBindings(normalized.Validators)
return nil
}
func (r *ValidatorChainRegistry) Validators(stage ModuleStage, module string) []ModuleBinding {
if r == nil {
return nil
}
chain := r.chains[validatorChainKey{stage: stage, module: strings.TrimSpace(module)}]
return cloneModuleBindings(chain)
}
func normalizeValidatorChainMapping(mapping ValidatorChainMapping) (ValidatorChainMapping, error) {
normalized := ValidatorChainMapping{
Stage: mapping.Stage,
Module: strings.TrimSpace(mapping.Module),
Validators: cloneModuleBindings(mapping.Validators),
}
switch normalized.Stage {
case StageChunk, StageExtract, StageMerge, StageNormalize:
default:
return ValidatorChainMapping{}, fmt.Errorf("validator chain stage %q is not supported", normalized.Stage)
}
if normalized.Module == "" {
return ValidatorChainMapping{}, fmt.Errorf("validator chain module key must not be empty")
}
for i, validator := range normalized.Validators {
if strings.TrimSpace(validator.Module) == "" {
return ValidatorChainMapping{}, fmt.Errorf("validator chain for %q %q has empty validator key at index %d", normalized.Stage, normalized.Module, i)
}
normalized.Validators[i] = resolveBinding(validator, "")
}
return normalized, nil
}
func cloneModuleBindings(bindings []ModuleBinding) []ModuleBinding {
if len(bindings) == 0 {
return nil
}
out := make([]ModuleBinding, len(bindings))
for i, binding := range bindings {
out[i] = cloneModuleBinding(binding)
}
return out
}
func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
binding.Module = strings.TrimSpace(binding.Module)
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
binding.Options = cloneOptions(binding.Options)
if len(binding.References) > 0 {
references := make(map[string]string, len(binding.References))
for key, value := range binding.References {
references[key] = value
}
binding.References = references
}
return binding
}

View File

@@ -0,0 +1,74 @@
package pipeline
import "testing"
func TestValidatorChainRegistryRegistersAndLooksUpChains(t *testing.T) {
registry := NewValidatorChainRegistry()
err := registry.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: " extractor ",
Validators: []ModuleBinding{{Module: " first "}, {Module: "second"}},
})
if err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
got := registry.Validators(StageExtract, " extractor ")
if len(got) != 2 || got[0].Module != "first" || got[1].Module != "second" {
t.Fatalf("Validators() = %#v, want trimmed chain", got)
}
}
func TestValidatorChainRegistryRejectsDuplicateMappings(t *testing.T) {
registry := NewValidatorChainRegistry()
mapping := ValidatorChainMapping{Stage: StageMerge, Module: "merge", Validators: []ModuleBinding{Binding("validator")}}
if err := registry.Register(mapping); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
if err := registry.Register(mapping); err == nil {
t.Fatal("Register() error = nil, want duplicate mapping error")
}
}
func TestValidatorChainRegistryRejectsUnsupportedStage(t *testing.T) {
registry := NewValidatorChainRegistry()
err := registry.Register(ValidatorChainMapping{Stage: StageInput, Module: "input"})
if err == nil {
t.Fatal("Register() error = nil, want unsupported stage error")
}
}
func TestValidatorChainRegistryAllowsAbsentAndEmptyChains(t *testing.T) {
registry := NewValidatorChainRegistry()
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
t.Fatalf("absent chain = %#v, want nil", got)
}
if err := registry.Register(ValidatorChainMapping{Stage: StageNormalize, Module: "normalize"}); err != nil {
t.Fatalf("Register(empty) error = %v, want nil", err)
}
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
t.Fatalf("empty chain = %#v, want nil", got)
}
}
func TestValidatorChainRegistryReturnsDefensiveCopies(t *testing.T) {
registry := NewValidatorChainRegistry()
err := registry.Register(ValidatorChainMapping{
Stage: StageChunk,
Module: "chunk",
Validators: []ModuleBinding{{Module: "validator", Options: map[string]any{"level": "strict"}}},
})
if err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
got := registry.Validators(StageChunk, "chunk")
got[0].Module = "changed"
got[0].Options["level"] = "changed"
again := registry.Validators(StageChunk, "chunk")
if again[0].Module != "validator" || again[0].Options["level"] != "strict" {
t.Fatalf("Validators() after caller mutation = %#v, want original chain", again)
}
}

View File

@@ -2,6 +2,7 @@ package pipeline
import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -9,29 +10,34 @@ import (
type ValidatorConstructor func() (contracts.Validator, error)
type ValidatorSpec struct {
Key string `json:"key"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
}
type ValidatorRegistry struct {
constructors map[string]ValidatorConstructor
specs map[string]ModuleSpec
specs map[string]ValidatorSpec
}
func NewValidatorRegistry() *ValidatorRegistry {
return &ValidatorRegistry{
constructors: make(map[string]ValidatorConstructor),
specs: make(map[string]ModuleSpec),
specs: make(map[string]ValidatorSpec),
}
}
func (r *ValidatorRegistry) Register(key string, constructor ValidatorConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageValidate), constructor)
return r.RegisterWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
}
func (r *ValidatorRegistry) RegisterWithSpec(spec ModuleSpec, constructor ValidatorConstructor) error {
func (r *ValidatorRegistry) RegisterWithSpec(spec ValidatorSpec, constructor ValidatorConstructor) error {
if r == nil {
return fmt.Errorf("validator registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("validator", StageValidate, normalizedSpec); err != nil {
normalizedSpec, err := normalizeValidatorSpec(spec)
if err != nil {
return err
}
if constructor == nil {
@@ -45,10 +51,10 @@ func (r *ValidatorRegistry) RegisterWithSpec(spec ModuleSpec, constructor Valida
r.constructors = make(map[string]ValidatorConstructor)
}
if r.specs == nil {
r.specs = make(map[string]ModuleSpec)
r.specs = make(map[string]ValidatorSpec)
}
r.constructors[normalizedSpec.Key] = constructor
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
r.specs[normalizedSpec.Key] = normalizedSpec
return nil
}
@@ -77,25 +83,45 @@ func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) {
if validator.Name() != normalizedKey {
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
}
switch validator.ExecutionClass() {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return nil, fmt.Errorf("validator %q returned unsupported execution class %q", normalizedKey, validator.ExecutionClass())
spec, ok := r.specs[normalizedKey]
if !ok {
return nil, fmt.Errorf("validator %q spec is not registered", normalizedKey)
}
if validator.ExecutionClass() != spec.ExecutionClass {
return nil, fmt.Errorf("validator %q returned execution class %q, want %q", normalizedKey, validator.ExecutionClass(), spec.ExecutionClass)
}
return validator, nil
}
func (r *ValidatorRegistry) Spec(key string) (ModuleSpec, bool) {
func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
if r == nil {
return ModuleSpec{}, false
return ValidatorSpec{}, false
}
spec, ok := r.specs[strings.TrimSpace(key)]
if !ok {
return ModuleSpec{}, false
return ValidatorSpec{}, false
}
return cloneModuleSpec(spec), true
return spec, true
}
func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec {
if r == nil || len(r.specs) == 0 {
return nil
}
keys := make([]string, 0, len(r.specs))
for key := range r.specs {
keys = append(keys, key)
}
sort.Strings(keys)
specs := make([]ValidatorSpec, 0, len(keys))
for _, key := range keys {
specs = append(specs, r.specs[key])
}
return specs
}
func (r *ValidatorRegistry) RegisteredKeys() []string {
@@ -105,3 +131,19 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
return sortedRegistryKeys(r.constructors)
}
func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
normalized := ValidatorSpec{
Key: strings.TrimSpace(spec.Key),
ExecutionClass: spec.ExecutionClass,
}
if normalized.Key == "" {
return ValidatorSpec{}, fmt.Errorf("validator key must not be empty")
}
switch normalized.ExecutionClass {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return ValidatorSpec{}, fmt.Errorf("validator %q execution class %q is not supported", normalized.Key, normalized.ExecutionClass)
}
return normalized, nil
}

View File

@@ -2,88 +2,116 @@ package pipeline
import (
"context"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestValidatorRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.Validator]{
name: "ValidatorRegistry",
key: "generic-validator",
stage: StageValidate,
wrongStage: StageExtract,
newRegistry: func() any {
return NewValidatorRegistry()
},
register: func(registry any, key string, constructor func() (contracts.Validator, error)) error {
return registry.(*ValidatorRegistry).Register(key, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.Validator, error)) error {
return registry.(*ValidatorRegistry).RegisterWithSpec(spec, constructor)
},
build: func(registry any, key string) (contracts.Validator, error) {
return registry.(*ValidatorRegistry).Build(key)
},
spec: func(registry any, key string) (ModuleSpec, bool) {
return registry.(*ValidatorRegistry).Spec(key)
},
registeredKeys: func(registry any) []string {
return registry.(*ValidatorRegistry).RegisteredKeys()
},
nilRegister: func(key string, constructor func() (contracts.Validator, error)) error {
var registry *ValidatorRegistry
return registry.Register(key, constructor)
},
nilBuild: func(key string) (contracts.Validator, error) {
var registry *ValidatorRegistry
return registry.Build(key)
},
nilSpec: func(key string) (ModuleSpec, bool) {
var registry *ValidatorRegistry
return registry.Spec(key)
},
nilRegisteredKey: func() []string {
var registry *ValidatorRegistry
return registry.RegisteredKeys()
},
constructor: func(key string) func() (contracts.Validator, error) {
return func() (contracts.Validator, error) {
return registryValidator{name: key}, nil
}
},
moduleKey: func(module contracts.Validator) string {
return module.Name()
},
})
registry := NewValidatorRegistry()
if err := registry.Register(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.Build("generic-validator")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if validator.Name() != "generic-validator" {
t.Fatalf("validator name = %q, want generic-validator", validator.Name())
}
spec, ok := registry.Spec(" generic-validator ")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ValidatorSpec{Key: "generic-validator", ExecutionClass: contracts.ExecutionClassDeterministic}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("Spec() = %#v, want %#v", spec, want)
}
}
func TestValidatorRegistryRegistersSpecs(t *testing.T) {
registry := NewValidatorRegistry()
spec := ValidatorSpec{Key: " llm-validator ", ExecutionClass: contracts.ExecutionClassLLMBacked}
if err := registry.RegisterWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
got, ok := registry.Spec("llm-validator")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ValidatorSpec{Key: "llm-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
}
func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
registry := NewValidatorRegistry()
for _, key := range []string{"zeta", "alpha"} {
if err := registry.Register(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
t.Fatalf("Register(%q) error = %v", key, err)
}
}
specs := registry.RegisteredSpecs()
if len(specs) != 2 || specs[0].Key != "alpha" || specs[1].Key != "zeta" {
t.Fatalf("RegisteredSpecs() = %#v, want sorted specs", specs)
}
}
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.Register("invalid-validator", func() (contracts.Validator, error) {
return invalidExecutionClassValidator{name: "invalid-validator"}, nil
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("invalid-validator")
err := registry.RegisterWithSpec(
ValidatorSpec{Key: "invalid-validator", ExecutionClass: contracts.ExecutionClass("unsupported")},
validatorConstructor("invalid-validator", contracts.ExecutionClass("unsupported")),
)
if err == nil {
t.Fatal("Build() error = nil, want unsupported execution class error")
t.Fatal("RegisterWithSpec() error = nil, want unsupported execution class error")
}
}
type invalidExecutionClassValidator struct {
name string
func TestValidatorRegistryRejectsConstructorExecutionClassMismatch(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.RegisterWithSpec(
ValidatorSpec{Key: "validator", ExecutionClass: contracts.ExecutionClassDeterministic},
validatorConstructor("validator", contracts.ExecutionClassLLMBacked),
); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
_, err := registry.Build("validator")
if err == nil {
t.Fatal("Build() error = nil, want execution class mismatch")
}
if !strings.Contains(err.Error(), "execution class") {
t.Fatalf("Build() error = %q, want execution class context", err.Error())
}
}
func (validator invalidExecutionClassValidator) Name() string {
type testValidator struct {
name string
executionClass contracts.ExecutionClass
}
func validatorConstructor(name string, executionClass contracts.ExecutionClass) ValidatorConstructor {
return func() (contracts.Validator, error) {
return testValidator{name: name, executionClass: executionClass}, nil
}
}
func (validator testValidator) Name() string {
return validator.name
}
func (validator invalidExecutionClassValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClass("unsupported")
func (validator testValidator) ExecutionClass() contracts.ExecutionClass {
return validator.executionClass
}
func (validator invalidExecutionClassValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
func (validator testValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -97,12 +97,13 @@ func walkingSkeletonCatalog(t *testing.T) ModuleCatalog {
t.Helper()
catalog := ModuleCatalog{
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
Outputs: NewOutputEncoderRegistry(),
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
ValidatorChains: NewValidatorChainRegistry(),
Outputs: NewOutputEncoderRegistry(),
}
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
Key: "fake/input",

View File

@@ -212,12 +212,13 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
}
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Outputs: outputs,
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
}
}

View File

@@ -151,12 +151,13 @@ func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.M
})
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Outputs: outputs,
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
}
}