Add validator chain provenance

This commit is contained in:
2026-07-07 21:10:29 +00:00
parent b7ad66f0e0
commit d593bfee0a
20 changed files with 845 additions and 324 deletions

View File

@@ -82,6 +82,14 @@ sanitizing the lane ID:
"normalizer": "noop" "normalizer": "noop"
} }
], ],
"validator_chains": [
{
"stage": "extract",
"lane_id": "spells",
"module_key": "dnd/spells",
"validators": []
}
],
"validation_status": "approved", "validation_status": "approved",
"started_at": "2026-01-01T00:00:00Z", "started_at": "2026-01-01T00:00:00Z",
"completed_at": "2026-01-01T00:00:01Z" "completed_at": "2026-01-01T00:00:01Z"
@@ -103,6 +111,11 @@ references.
`validation_status` is `approved` when no raw outputs were rejected and `validation_status` is `approved` when no raw outputs were rejected and
`rejected` when one or more raw outputs were rejected. `rejected` when one or more raw outputs were rejected.
`validator_chains` records the resolved validator chain for each validation
point. Entries include stage, lane ID when applicable, module key, and validators
with key and execution class. Empty chains are recorded with an empty
`validators` array.
`normalized_outputs` summarizes each normalized lane output without embedding `normalized_outputs` summarizes each normalized lane output without embedding
payload bytes. Entries include lane ID, normalizer module key, source ID, media payload bytes. Entries include lane ID, normalizer module key, source ID, media
type, and response schema provenance where available. type, and response schema provenance where available.

View File

@@ -3,17 +3,15 @@
This roadmap defines the target state for making validation a first-class, This roadmap defines the target state for making validation a first-class,
composable pipeline concern. composable pipeline concern.
Current pipeline behavior is raw-output based. The runner can execute Current pipeline behavior is raw-output based. The runner validates `chunk`,
`contracts.RawValidator` chains from `pipeline.RawValidationRegistry` for `extract`, `merge`, and `normalize` outputs through `contracts.Validator`
`chunk`, `extract`, `merge`, and `normalize` outputs. Empty chains approve by chains resolved from `pipeline.ValidatorChainRegistry`. Empty chains approve by
default, validator rejection records a rejected raw output, and rejected output default, validator rejection records a rejected raw output, and rejected output
does not pass to the next stage. Production currently registers no raw does not pass to the next stage. Production currently registers no validators,
validators, and non-empty pipeline-configured validator lists are rejected so and non-empty pipeline-configured validator lists are rejected until
they cannot appear in manifests without executing. stage-scoped override syntax is implemented.
Legacy candidate validator contracts and D&D spell validators still exist under The desired end state is that validator implementations, validator
`internal/modules/extract/dnd/spells`, but they are not part of the current
runner path. The desired end state is that validator implementations, validator
registration, and default module-to-validator mappings are explicit, reviewable, registration, and default module-to-validator mappings are explicit, reviewable,
and independent of concrete module packages. and independent of concrete module packages.
@@ -21,9 +19,8 @@ and independent of concrete module packages.
- Move artifact and module-output validation behavior out of `internal/modules` - Move artifact and module-output validation behavior out of `internal/modules`
and into `internal/validators`. and into `internal/validators`.
- Retire or replace the legacy candidate-oriented `contracts.Validator`, - Keep the raw module-output `contracts.Validator`, `ValidationRequest`, and
`ValidationRequest`, and `ValidationResult` path after equivalent raw-output `ValidationResult` path as the single framework validator contract.
validators exist.
- Keep each validator in its own package. - Keep each validator in its own package.
- Mirror the stage and domain shape of `internal/modules` where a validator is - Mirror the stage and domain shape of `internal/modules` where a validator is
module-specific. module-specific.
@@ -261,11 +258,10 @@ The validator framework should support validation of outputs from `chunk`,
enough for stage-specific validators to inspect the output they care about while enough for stage-specific validators to inspect the output they care about while
ignoring irrelevant fields. ignoring irrelevant fields.
The current `contracts.RawValidationRequest` is the right starting point. It The current `contracts.ValidationRequest` carries stage, lane, module, source,
already carries stage, lane, module, source, source and chunk provenance, source and chunk provenance, response schema metadata, raw payload, and run
response schema metadata, raw payload, and run metadata. The final contract metadata. The final contract should continue evolving from that raw-output
should evolve from that raw-output shape rather than from the legacy shape.
artifact-candidate `ValidationRequest`.
Additional fields needed for the full validator system include: Additional fields needed for the full validator system include:

View File

@@ -20,13 +20,14 @@ import (
func productionRegistries() (pipeline.Registries, error) { func productionRegistries() (pipeline.Registries, error) {
registries := pipeline.Registries{ registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(), Inputs: pipeline.NewInputAdapterRegistry(),
Chunkers: pipeline.NewChunkerRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
Extractors: pipeline.NewExtractorRegistry(), Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(), Mergers: pipeline.NewMergerRegistry(),
Normalizers: pipeline.NewNormalizerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(), Validators: pipeline.NewValidatorRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
} }
if err := seriatim.Register(registries.Inputs); err != nil { if err := seriatim.Register(registries.Inputs); err != nil {
return pipeline.Registries{}, fmt.Errorf("register seriatim input: %w", err) return pipeline.Registries{}, fmt.Errorf("register seriatim input: %w", err)
@@ -93,25 +94,27 @@ func effectiveRegistries(opts Options) (pipeline.Registries, error) {
func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalog { func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalog {
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: registries.Inputs, Inputs: registries.Inputs,
Chunkers: registries.Chunkers, Chunkers: registries.Chunkers,
Extractors: registries.Extractors, Extractors: registries.Extractors,
Mergers: registries.Mergers, Mergers: registries.Mergers,
Normalizers: registries.Normalizers, Normalizers: registries.Normalizers,
Validators: registries.Validators, Validators: registries.Validators,
Outputs: registries.Outputs, ValidatorChains: registries.ValidatorChains,
Outputs: registries.Outputs,
} }
} }
func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries { func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
return pipeline.Registries{ return pipeline.Registries{
Inputs: catalog.Inputs, Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers, Chunkers: catalog.Chunkers,
Extractors: catalog.Extractors, Extractors: catalog.Extractors,
Mergers: catalog.Mergers, Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers, Normalizers: catalog.Normalizers,
Validators: catalog.Validators, Validators: catalog.Validators,
Outputs: catalog.Outputs, ValidatorChains: catalog.ValidatorChains,
Outputs: catalog.Outputs,
} }
} }
@@ -122,6 +125,7 @@ func isEmptyCatalog(catalog pipeline.ModuleCatalog) bool {
catalog.Mergers == nil && catalog.Mergers == nil &&
catalog.Normalizers == nil && catalog.Normalizers == nil &&
catalog.Validators == nil && catalog.Validators == nil &&
catalog.ValidatorChains == nil &&
catalog.Outputs == nil catalog.Outputs == nil
} }
@@ -132,6 +136,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
registries.Mergers == nil && registries.Mergers == nil &&
registries.Normalizers == nil && registries.Normalizers == nil &&
registries.Validators == nil && registries.Validators == nil &&
registries.ValidatorChains == nil &&
registries.Outputs == nil registries.Outputs == nil
} }

View File

@@ -2931,13 +2931,14 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
mustRegisterOutput(t, outputs, specs["json"]) mustRegisterOutput(t, outputs, specs["json"])
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
Validators: validators, Validators: validators,
Outputs: outputs, ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
} }
} }

View File

@@ -38,10 +38,21 @@ type ArtifactLaneManifest struct {
Extractor string `json:"extractor"` Extractor string `json:"extractor"`
Merger string `json:"merger"` Merger string `json:"merger"`
Normalizer string `json:"normalizer"` Normalizer string `json:"normalizer"`
Validators []string `json:"validators,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
} }
type ValidatorChainManifest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Validators []ValidatorManifest `json:"validators"`
}
type ValidatorManifest struct {
Key string `json:"key"`
ExecutionClass string `json:"execution_class"`
}
type LLMProfileManifest struct { type LLMProfileManifest struct {
ID string `json:"id"` ID string `json:"id"`
Provider string `json:"provider,omitempty"` Provider string `json:"provider,omitempty"`
@@ -100,6 +111,7 @@ type RunManifest struct {
OutputEncoder string `json:"output_encoder,omitempty"` OutputEncoder string `json:"output_encoder,omitempty"`
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"` ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"` ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
ValidatorChains []ValidatorChainManifest `json:"validator_chains,omitempty"`
References []ReferenceProvenance `json:"references,omitempty"` References []ReferenceProvenance `json:"references,omitempty"`
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"` NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"` RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`

View File

@@ -136,12 +136,21 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
Extractor: "event-extractor", Extractor: "event-extractor",
Merger: "appendorder", Merger: "appendorder",
Normalizer: "noop", Normalizer: "noop",
Validators: []string{"grounded"},
Metadata: map[string]any{ Metadata: map[string]any{
"extractor": map[string]any{"prompt_id": "test.prompt"}, "extractor": map[string]any{"prompt_id": "test.prompt"},
}, },
}, },
}, },
ValidatorChains: []ValidatorChainManifest{
{
Stage: "extract",
LaneID: "events",
ModuleKey: "event-extractor",
Validators: []ValidatorManifest{
{Key: "grounded", ExecutionClass: "deterministic"},
},
},
},
} }
gotJSON, err := json.Marshal(manifest) gotJSON, err := json.Marshal(manifest)
@@ -154,7 +163,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
t.Fatalf("json.Unmarshal() error = %v", err) t.Fatalf("json.Unmarshal() error = %v", err)
} }
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles") assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "validator_chains", "llm_profiles")
profiles, ok := got["llm_profiles"].([]any) profiles, ok := got["llm_profiles"].([]any)
if !ok { if !ok {
@@ -180,7 +189,20 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
if !ok { if !ok {
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0]) t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
} }
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata") assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "metadata")
chains, ok := got["validator_chains"].([]any)
if !ok {
t.Fatalf("validator_chains = %#v, want array", got["validator_chains"])
}
if len(chains) != 1 {
t.Fatalf("len(validator_chains) = %d, want 1", len(chains))
}
chain, ok := chains[0].(map[string]any)
if !ok {
t.Fatalf("validator_chains[0] = %#v, want object", chains[0])
}
assertHasKeys(t, chain, "stage", "lane_id", "module_key", "validators")
} }
func TestRunManifestIncludesReferenceProvenance(t *testing.T) { func TestRunManifestIncludesReferenceProvenance(t *testing.T) {

View File

@@ -429,13 +429,14 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
mustRegisterOutput(t, outputs, specs["json"]) mustRegisterOutput(t, outputs, specs["json"])
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
Validators: validators, Validators: validators,
Outputs: outputs, ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
} }
} }
@@ -476,7 +477,8 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry,
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) { func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Validator, error) { return nil, nil }); err != nil { validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
t.Fatalf("register validator: %v", err) t.Fatalf("register validator: %v", err)
} }
} }

View File

@@ -92,12 +92,13 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
} }
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
Outputs: outputs, ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
} }
} }

View File

@@ -83,6 +83,18 @@ type ResolvedArtifactLane struct {
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"` NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
} }
type ResolvedValidatorChain struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Validators []ResolvedValidator `json:"validators"`
}
type ResolvedValidator struct {
Binding ModuleBinding `json:"binding"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
}
type ResolvedPipeline struct { type ResolvedPipeline struct {
ID string ID string
Digest string Digest string
@@ -90,17 +102,19 @@ type ResolvedPipeline struct {
Chunk ModuleBinding Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"` ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane ArtifactLanes []ResolvedArtifactLane
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
Output ModuleBinding Output ModuleBinding
} }
type ModuleCatalog struct { type ModuleCatalog struct {
Inputs *InputAdapterRegistry Inputs *InputAdapterRegistry
Chunkers *ChunkerRegistry Chunkers *ChunkerRegistry
Extractors *ExtractorRegistry Extractors *ExtractorRegistry
Mergers *MergerRegistry Mergers *MergerRegistry
Normalizers *NormalizerRegistry Normalizers *NormalizerRegistry
Validators *ValidatorRegistry Validators *ValidatorRegistry
Outputs *OutputEncoderRegistry ValidatorChains *ValidatorChainRegistry
Outputs *OutputEncoderRegistry
} }
func Binding(module string) ModuleBinding { func Binding(module string) ModuleBinding {
@@ -172,15 +186,21 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences), ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule), Output: resolveBinding(profile.Output, DefaultOutputModule),
} }
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
resolved.ValidatorChains = append(resolved.ValidatorChains, chunkValidatorChain)
outputCapabilities := capabilities.clone() outputCapabilities := capabilities.clone()
for _, laneID := range selectedLaneIDs { for _, laneID := range selectedLaneIDs {
laneProfile := lanesByID[laneID] laneProfile := lanesByID[laneID]
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog) lane, validatorChains, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
if err != nil { if err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane) resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
resolved.ValidatorChains = append(resolved.ValidatorChains, validatorChains...)
outputCapabilities.addSet(laneCapabilities) outputCapabilities.addSet(laneCapabilities)
} }
@@ -208,7 +228,7 @@ func resolveArtifactLane(
options ResolveOptions, options ResolveOptions,
inherited capabilitySet, inherited capabilitySet,
catalog ModuleCatalog, catalog ModuleCatalog,
) (ResolvedArtifactLane, capabilitySet, error) { ) (ResolvedArtifactLane, []ResolvedValidatorChain, capabilitySet, error) {
lane := ResolvedArtifactLane{ lane := ResolvedArtifactLane{
ID: laneID, ID: laneID,
Extract: resolveBinding(profile.Extract, ""), Extract: resolveBinding(profile.Extract, ""),
@@ -217,17 +237,17 @@ func resolveArtifactLane(
Validators: resolveBindings(profile.Validators, ""), Validators: resolveBindings(profile.Validators, ""),
} }
if lane.Extract.Module == "" { if lane.Extract.Module == "" {
return ResolvedArtifactLane{}, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID) return ResolvedArtifactLane{}, nil, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
} }
capabilities := inherited.clone() capabilities := inherited.clone()
extractSpec, err := extractorSpec(catalog, lane.Extract.Module) extractSpec, err := extractorSpec(catalog, lane.Extract.Module)
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err) return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
} }
if missing, ok := capabilities.missing(extractSpec.Requires); ok { if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing) return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
} }
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References) extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{ references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
@@ -241,17 +261,17 @@ func resolveArtifactLane(
Options: options, Options: options,
}) })
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, err return ResolvedArtifactLane{}, nil, nil, err
} }
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references) lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...) capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module) mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err) return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
} }
if missing, ok := capabilities.missing(mergeSpec.Requires); ok { if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing) return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
} }
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{ mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID, PipelineID: pipelineID,
@@ -264,17 +284,17 @@ func resolveArtifactLane(
Options: options, Options: options,
}) })
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, err return ResolvedArtifactLane{}, nil, nil, err
} }
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences) lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
capabilities.add(mergeSpec.Provides...) capabilities.add(mergeSpec.Provides...)
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module) normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err) return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
} }
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok { if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing) return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
} }
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{ normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID, PipelineID: pipelineID,
@@ -287,22 +307,105 @@ func resolveArtifactLane(
Options: options, Options: options,
}) })
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, err return ResolvedArtifactLane{}, nil, nil, err
} }
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences) lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
capabilities.add(normalizeSpec.Provides...) capabilities.add(normalizeSpec.Provides...)
if len(lane.Validators) > 0 { if len(lane.Validators) > 0 {
return ResolvedArtifactLane{}, nil, configuredValidatorsError(pipelineID, laneID) return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
} }
return lane, capabilities, nil extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
validatorChains := []ResolvedValidatorChain{extractValidatorChain, mergeValidatorChain, normalizeValidatorChain}
return lane, validatorChains, capabilities, nil
} }
func configuredValidatorsError(pipelineID string, laneID string) error { func configuredValidatorsError(pipelineID string, laneID string) error {
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID) return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID)
} }
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
chain := ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
ModuleKey: strings.TrimSpace(module),
}
if chain.ModuleKey == "" {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain %q module key must not be empty", pipelineID, stage)
}
switch stage {
case StageChunk, StageExtract, StageMerge, StageNormalize:
default:
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
}
if catalog.ValidatorChains == nil {
return chain, nil
}
bindings := catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
if len(bindings) == 0 {
return chain, nil
}
if catalog.Validators == nil {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator registry must not be nil for %s validator chain on module %q", pipelineID, stage, chain.ModuleKey)
}
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
for _, validator := range bindings {
spec, ok := catalog.Validators.Spec(validator.Module)
if !ok {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
}
chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(validator),
ExecutionClass: spec.ExecutionClass,
})
}
return chain, nil
}
func cloneResolvedValidatorChains(chains []ResolvedValidatorChain) []ResolvedValidatorChain {
if len(chains) == 0 {
return nil
}
out := make([]ResolvedValidatorChain, len(chains))
for i, chain := range chains {
out[i] = ResolvedValidatorChain{
Stage: chain.Stage,
LaneID: strings.TrimSpace(chain.LaneID),
ModuleKey: strings.TrimSpace(chain.ModuleKey),
Validators: cloneResolvedValidators(chain.Validators),
}
}
return out
}
func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator {
if len(validators) == 0 {
return nil
}
out := make([]ResolvedValidator, len(validators))
for i, validator := range validators {
out[i] = ResolvedValidator{
Binding: cloneModuleBinding(validator.Binding),
ExecutionClass: validator.ExecutionClass,
}
}
return out
}
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget { func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
return ResolvedReferenceTarget{ return ResolvedReferenceTarget{
Stage: stage, Stage: stage,

View File

@@ -110,6 +110,78 @@ func TestResolvePipelineAppliesDefaults(t *testing.T) {
} }
} }
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: []ModuleBinding{Binding("grounded")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
resolved, err := ResolvePipeline(PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {Extract: Binding("event-extractor")},
},
}, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if len(resolved.ValidatorChains) != 4 {
t.Fatalf("len(ValidatorChains) = %d, want chunk plus lane extract/merge/normalize", len(resolved.ValidatorChains))
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
if extractChain == nil {
t.Fatal("extract validator chain not found")
}
if len(extractChain.Validators) != 1 {
t.Fatalf("extract validators = %#v, want one validator", extractChain.Validators)
}
if extractChain.Validators[0].Binding.Module != "grounded" {
t.Fatalf("extract validator key = %q, want grounded", extractChain.Validators[0].Binding.Module)
}
if extractChain.Validators[0].ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("extract validator execution class = %q, want deterministic", extractChain.Validators[0].ExecutionClass)
}
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", DefaultChunkModule)
if chunkChain == nil {
t.Fatal("chunk validator chain not found")
}
if len(chunkChain.Validators) != 0 {
t.Fatalf("chunk validators = %#v, want explicit empty chain", chunkChain.Validators)
}
}
func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageNormalize,
Module: DefaultNormalizeModule,
Validators: []ModuleBinding{Binding("missing-validator")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
_, err := ResolvePipeline(PipelineProfile{
ID: "invalid-chain",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {Extract: Binding("event-extractor")},
},
}, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
}
if !strings.Contains(err.Error(), "missing-validator") {
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
}
}
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) { func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
profile := multiLaneProfile() profile := multiLaneProfile()
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t)) resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
@@ -955,6 +1027,15 @@ func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName str
t.Fatalf("binding %q not found in %#v", slotName, bindings) t.Fatalf("binding %q not found in %#v", slotName, bindings)
} }
func findResolvedValidatorChain(chains []ResolvedValidatorChain, stage ModuleStage, laneID string, module string) *ResolvedValidatorChain {
for i := range chains {
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
return &chains[i]
}
}
return nil
}
func newProfileCatalog(t *testing.T) ModuleCatalog { func newProfileCatalog(t *testing.T) ModuleCatalog {
t.Helper() t.Helper()
@@ -994,13 +1075,14 @@ func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) Modul
func emptyProfileCatalog() ModuleCatalog { func emptyProfileCatalog() ModuleCatalog {
return ModuleCatalog{ return ModuleCatalog{
Inputs: NewInputAdapterRegistry(), Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(), Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(), Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(), Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(), Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(), Validators: NewValidatorRegistry(),
Outputs: NewOutputEncoderRegistry(), ValidatorChains: NewValidatorChainRegistry(),
Outputs: NewOutputEncoderRegistry(),
} }
} }
@@ -1043,7 +1125,8 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
t.Fatalf("register normalizer spec %#v: %v", spec, err) t.Fatalf("register normalizer spec %#v: %v", spec, err)
} }
case StageValidate: case StageValidate:
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil { validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
if err := catalog.Validators.RegisterWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err) t.Fatalf("register validator spec %#v: %v", spec, err)
} }
case StageOutput: case StageOutput:

View File

@@ -1,78 +0,0 @@
package pipeline
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type rawValidationKey struct {
stage ModuleStage
module string
}
type RawValidationRegistry struct {
chains map[rawValidationKey][]contracts.Validator
}
func NewRawValidationRegistry() *RawValidationRegistry {
return &RawValidationRegistry{
chains: make(map[rawValidationKey][]contracts.Validator),
}
}
func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.Validator) error {
if r == nil {
return fmt.Errorf("raw validation registry must not be nil")
}
normalizedModule := strings.TrimSpace(module)
if normalizedModule == "" {
return fmt.Errorf("raw validation module key must not be empty")
}
switch stage {
case StageChunk, StageExtract, StageMerge, StageNormalize:
default:
return fmt.Errorf("raw validation stage %q is not supported", stage)
}
if len(validators) == 0 {
return fmt.Errorf("raw validation chain for %q %q must not be empty", stage, normalizedModule)
}
chain := make([]contracts.Validator, 0, len(validators))
for i, validator := range validators {
if validator == nil {
return fmt.Errorf("raw validator %d for %q %q must not be nil", i, stage, normalizedModule)
}
if strings.TrimSpace(validator.Name()) == "" {
return fmt.Errorf("raw validator %d for %q %q must not have an empty name", i, stage, normalizedModule)
}
switch validator.ExecutionClass() {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return fmt.Errorf("raw validator %q for %q %q has unsupported execution class %q", validator.Name(), stage, normalizedModule, validator.ExecutionClass())
}
chain = append(chain, validator)
}
if r.chains == nil {
r.chains = make(map[rawValidationKey][]contracts.Validator)
}
key := rawValidationKey{stage: stage, module: normalizedModule}
if _, exists := r.chains[key]; exists {
return fmt.Errorf("raw validation chain for %q %q is already registered", stage, normalizedModule)
}
r.chains[key] = append([]contracts.Validator(nil), chain...)
return nil
}
func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.Validator {
if r == nil {
return nil
}
chain := r.chains[rawValidationKey{stage: stage, module: strings.TrimSpace(module)}]
if len(chain) == 0 {
return nil
}
return append([]contracts.Validator(nil), chain...)
}

View File

@@ -18,14 +18,14 @@ import (
) )
type Registries struct { type Registries struct {
Inputs *InputAdapterRegistry Inputs *InputAdapterRegistry
Chunkers *ChunkerRegistry Chunkers *ChunkerRegistry
Extractors *ExtractorRegistry Extractors *ExtractorRegistry
Mergers *MergerRegistry Mergers *MergerRegistry
Normalizers *NormalizerRegistry Normalizers *NormalizerRegistry
Validators *ValidatorRegistry Validators *ValidatorRegistry
RawValidators *RawValidationRegistry ValidatorChains *ValidatorChainRegistry
Outputs *OutputEncoderRegistry Outputs *OutputEncoderRegistry
} }
type Runner struct { type Runner struct {
@@ -127,7 +127,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err != nil { if err != nil {
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err) return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
} }
rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, input.Metadata, attempt) rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, input.Metadata, input.Pipeline.ValidatorChains, attempt)
if err != nil || rejection != nil { if err != nil || rejection != nil {
return false, rejection, err return false, rejection, err
} }
@@ -240,6 +240,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
schema: extractOutput.Schema, schema: extractOutput.Schema,
payload: extractOutput.Payload, payload: extractOutput.Payload,
metadata: input.Metadata, metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt, attempt: attempt,
}) })
if err != nil || rejection != nil { if err != nil || rejection != nil {
@@ -296,6 +297,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
schema: mergeOutput.Schema, schema: mergeOutput.Schema,
payload: mergeOutput.Payload, payload: mergeOutput.Payload,
metadata: input.Metadata, metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt, attempt: attempt,
}) })
if err != nil || rejection != nil { if err != nil || rejection != nil {
@@ -346,6 +348,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
schema: normalizeOutput.Schema, schema: normalizeOutput.Schema,
payload: normalizeOutput.Payload, payload: normalizeOutput.Payload,
metadata: input.Metadata, metadata: input.Metadata,
chains: input.Pipeline.ValidatorChains,
attempt: attempt, attempt: attempt,
}) })
if err != nil || rejection != nil { if err != nil || rejection != nil {
@@ -378,6 +381,7 @@ type rawValidationTarget struct {
schema contracts.ResponseSchema schema contracts.ResponseSchema
payload contracts.RawPayload payload contracts.RawPayload
metadata map[string]any metadata map[string]any
chains []ResolvedValidatorChain
attempt int attempt int
} }
@@ -428,7 +432,7 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
return false, lastRejection, nil return false, lastRejection, nil
} }
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, metadata map[string]any, attempt int) (*contracts.RejectedOutput, error) { func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, metadata map[string]any, chains []ResolvedValidatorChain, attempt int) (*contracts.RejectedOutput, error) {
for _, chunk := range chunks { for _, chunk := range chunks {
_, rejection, err := r.validateRaw(ctx, rawValidationTarget{ _, rejection, err := r.validateRaw(ctx, rawValidationTarget{
stage: StageChunk, stage: StageChunk,
@@ -443,6 +447,7 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
Metadata: cloneMetadata(chunk.Metadata), Metadata: cloneMetadata(chunk.Metadata),
}, },
metadata: metadata, metadata: metadata,
chains: chains,
attempt: attempt, attempt: attempt,
}) })
if err != nil { if err != nil {
@@ -456,10 +461,16 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
} }
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) { func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
validators := r.registries.RawValidators.Validators(target.stage, target.moduleKey) chain := resolvedValidatorChain(target.stage, target.laneID, target.moduleKey, target.chains)
if len(validators) == 0 { if len(chain.Validators) == 0 {
chain = r.registryValidatorChain(target.stage, target.laneID, target.moduleKey)
}
if len(chain.Validators) == 0 {
return nil, nil, nil return nil, nil, nil
} }
if r.registries.Validators == nil {
return nil, nil, fmt.Errorf("validator registry must not be nil")
}
request := contracts.ValidationRequest{ request := contracts.ValidationRequest{
Stage: string(target.stage), Stage: string(target.stage),
@@ -475,7 +486,11 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
} }
var warnings []contracts.Warning var warnings []contracts.Warning
for _, validator := range validators { for _, validatorBinding := range chain.Validators {
validator, err := r.registries.Validators.Build(validatorBinding.Binding.Module)
if err != nil {
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
}
result, err := validator.Validate(ctx, request) result, err := validator.Validate(ctx, request)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err) return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
@@ -507,6 +522,60 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
return warnings, nil, nil return warnings, nil, nil
} }
func (r *Runner) registryValidatorChain(stage ModuleStage, laneID string, moduleKey string) ResolvedValidatorChain {
chain := ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
ModuleKey: strings.TrimSpace(moduleKey),
}
if r == nil || r.registries.ValidatorChains == nil {
return chain
}
bindings := r.registries.ValidatorChains.Validators(stage, moduleKey)
if len(bindings) == 0 {
return chain
}
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
for _, binding := range bindings {
executionClass := contracts.ExecutionClass("")
if r.registries.Validators != nil {
if spec, ok := r.registries.Validators.Spec(binding.Module); ok {
executionClass = spec.ExecutionClass
}
}
chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(binding),
ExecutionClass: executionClass,
})
}
return chain
}
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
for _, chain := range chains {
if chain.Stage != stage {
continue
}
if chain.ModuleKey != moduleKey {
continue
}
if strings.TrimSpace(chain.LaneID) != strings.TrimSpace(laneID) {
continue
}
return ResolvedValidatorChain{
Stage: chain.Stage,
LaneID: chain.LaneID,
ModuleKey: chain.ModuleKey,
Validators: cloneResolvedValidators(chain.Validators),
}
}
return ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
ModuleKey: strings.TrimSpace(moduleKey),
}
}
func (r *Runner) validateRegistries(pipeline ResolvedPipeline) error { func (r *Runner) validateRegistries(pipeline ResolvedPipeline) error {
if r.registries.Inputs == nil { if r.registries.Inputs == nil {
return fmt.Errorf("input registry must not be nil") return fmt.Errorf("input registry must not be nil")
@@ -580,16 +649,17 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
pipeline := input.Pipeline pipeline := input.Pipeline
manifest := artifacts.RunManifest{ manifest := artifacts.RunManifest{
PipelineID: pipeline.ID, PipelineID: pipeline.ID,
PipelineDigest: pipeline.Digest, PipelineDigest: pipeline.Digest,
InputModule: pipeline.Input.Module, InputModule: pipeline.Input.Module,
Chunker: pipeline.Chunk.Module, Chunker: pipeline.Chunk.Module,
OutputEncoder: pipeline.Output.Module, OutputEncoder: pipeline.Output.Module,
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)), ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
RunID: runID, ValidatorChains: validatorChainManifests(pipeline.ValidatorChains),
StartedAt: timePtr(startedAt), RunID: runID,
References: ReferenceProvenance(pipeline), StartedAt: timePtr(startedAt),
LLMProfiles: cloneLLMProfiles(input.LLMProfiles), References: ReferenceProvenance(pipeline),
LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
} }
// The runner does not currently maintain a cache or idempotency key. Reference // The runner does not currently maintain a cache or idempotency key. Reference
// digests are recorded in manifest provenance and intentionally kept separate // digests are recorded in manifest provenance and intentionally kept separate
@@ -607,6 +677,29 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
return manifest return manifest
} }
func validatorChainManifests(chains []ResolvedValidatorChain) []artifacts.ValidatorChainManifest {
if len(chains) == 0 {
return nil
}
manifests := make([]artifacts.ValidatorChainManifest, 0, len(chains))
for _, chain := range chains {
manifest := artifacts.ValidatorChainManifest{
Stage: string(chain.Stage),
LaneID: chain.LaneID,
ModuleKey: chain.ModuleKey,
Validators: make([]artifacts.ValidatorManifest, 0, len(chain.Validators)),
}
for _, validator := range chain.Validators {
manifest.Validators = append(manifest.Validators, artifacts.ValidatorManifest{
Key: validator.Binding.Module,
ExecutionClass: string(validator.ExecutionClass),
})
}
manifests = append(manifests, manifest)
}
return manifests
}
func failOutput(output RunOutput) RunOutput { func failOutput(output RunOutput) RunOutput {
if output.Manifest.PipelineID != "" { if output.Manifest.PipelineID != "" {
populateRawOutputManifest(&output) populateRawOutputManifest(&output)

View File

@@ -887,8 +887,9 @@ func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) { func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"} validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"}
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator) modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil { if err != nil {
@@ -909,8 +910,9 @@ func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) { func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"} validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator) modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil { if err != nil {
@@ -933,8 +935,9 @@ func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) { func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-merge", approved: []bool{false}, reason: "bad_merge", message: "merge rejected"} validator := &runnerChainValidator{name: "chain-merge", approved: []bool{false}, reason: "bad_merge", message: "merge rejected"}
modules.rawValidators = rawValidationRegistry(t, StageMerge, "merge", validator) modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageMerge, "merge", validator)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil { if err != nil {
@@ -954,8 +957,9 @@ func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
func TestRunRejectedNormalizePreventsOutputForLane(t *testing.T) { func TestRunRejectedNormalizePreventsOutputForLane(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-normalize", approved: []bool{false}, reason: "bad_normalize", message: "normalize rejected"} validator := &runnerChainValidator{name: "chain-normalize", approved: []bool{false}, reason: "bad_normalize", message: "normalize rejected"}
modules.rawValidators = rawValidationRegistry(t, StageNormalize, "normalize", validator) modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageNormalize, "normalize", validator)
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()}) output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil { if err != nil {
@@ -999,8 +1003,9 @@ func TestRunRetriesSameModuleInputAfterFrameworkError(t *testing.T) {
func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) { func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false, true, true}, reason: "bad_extract", message: "extract rejected"} validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true, true}, reason: "bad_extract", message: "extract rejected"}
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator) modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
pipeline := resolvedPipeline() pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].Extract.Retries = 1 pipeline.ArtifactLanes[0].Extract.Retries = 1
@@ -1023,8 +1028,9 @@ func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) { func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"} validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator) modules.validators[validator.name] = validator
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
pipeline := resolvedPipeline() pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].Extract.Retries = 1 pipeline.ArtifactLanes[0].Extract.Retries = 1
@@ -1312,8 +1318,11 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
if lane.ID != "alpha" || lane.Extractor != "extract-alpha" || lane.Merger != "merge" || lane.Normalizer != "normalize" { if lane.ID != "alpha" || lane.Extractor != "extract-alpha" || lane.Merger != "merge" || lane.Normalizer != "normalize" {
t.Fatalf("ArtifactLanes[0] = %#v, want lane details", lane) t.Fatalf("ArtifactLanes[0] = %#v, want lane details", lane)
} }
if len(lane.Validators) != 0 { if len(manifest.ValidatorChains) != 4 {
t.Fatalf("lane validators = %#v, want none", lane.Validators) t.Fatalf("ValidatorChains = %#v, want four validation points", manifest.ValidatorChains)
}
if manifest.ValidatorChains[0].Stage != string(StageChunk) || manifest.ValidatorChains[0].ModuleKey != "chunk" || len(manifest.ValidatorChains[0].Validators) != 0 {
t.Fatalf("chunk validator chain = %#v, want explicit empty chunk chain", manifest.ValidatorChains[0])
} }
} }
@@ -1454,6 +1463,12 @@ func resolvedPipeline() ResolvedPipeline {
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil), NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
}, },
}, },
ValidatorChains: []ResolvedValidatorChain{
{Stage: StageChunk, ModuleKey: "chunk"},
{Stage: StageExtract, LaneID: "alpha", ModuleKey: "extract-alpha"},
{Stage: StageMerge, LaneID: "alpha", ModuleKey: "merge"},
{Stage: StageNormalize, LaneID: "alpha", ModuleKey: "normalize"},
},
Output: Binding("output"), Output: Binding("output"),
} }
} }
@@ -1493,8 +1508,8 @@ type runnerModules struct {
extractors map[string]*runnerExtractor extractors map[string]*runnerExtractor
mergers map[string]*runnerMerger mergers map[string]*runnerMerger
normalizers map[string]*runnerNormalizer normalizers map[string]*runnerNormalizer
validators map[string]*runnerValidator validators map[string]contracts.Validator
rawValidators *RawValidationRegistry validatorChains *ValidatorChainRegistry
output *runnerOutputEncoder output *runnerOutputEncoder
inputBuildErr error inputBuildErr error
chunkerBuildErr error chunkerBuildErr error
@@ -1513,9 +1528,9 @@ func defaultRunnerModules() *runnerModules {
normalizers: map[string]*runnerNormalizer{ normalizers: map[string]*runnerNormalizer{
"normalize": {key: "normalize"}, "normalize": {key: "normalize"},
}, },
validators: map[string]*runnerValidator{ validators: map[string]contracts.Validator{
"configured": {name: "configured"}, "configured": &runnerValidator{name: "configured"},
"second-validator": {name: "second-validator"}, "second-validator": &runnerValidator{name: "second-validator"},
}, },
output: &runnerOutputEncoder{ output: &runnerOutputEncoder{
key: "output", key: "output",
@@ -1533,14 +1548,14 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
} }
registries := Registries{ registries := Registries{
Inputs: NewInputAdapterRegistry(), Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(), Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(), Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(), Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(), Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(), Validators: NewValidatorRegistry(),
RawValidators: modules.rawValidators, ValidatorChains: modules.validatorChains,
Outputs: NewOutputEncoderRegistry(), Outputs: NewOutputEncoderRegistry(),
} }
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) { if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
if modules.inputBuildErr != nil { if modules.inputBuildErr != nil {
@@ -1807,7 +1822,7 @@ type runnerValidator struct {
requests []contracts.ValidationRequest requests []contracts.ValidationRequest
} }
type runnerRawValidator struct { type runnerChainValidator struct {
name string name string
approved []bool approved []bool
reason string reason string
@@ -1818,15 +1833,15 @@ type runnerRawValidator struct {
requests []contracts.ValidationRequest requests []contracts.ValidationRequest
} }
func (validator *runnerRawValidator) Name() string { func (validator *runnerChainValidator) Name() string {
return validator.name return validator.name
} }
func (validator *runnerRawValidator) ExecutionClass() contracts.ExecutionClass { func (validator *runnerChainValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic return contracts.ExecutionClassDeterministic
} }
func (validator *runnerRawValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { func (validator *runnerChainValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
validator.calls++ validator.calls++
validator.requests = append(validator.requests, req) validator.requests = append(validator.requests, req)
if validator.err != nil { if validator.err != nil {
@@ -2037,12 +2052,16 @@ func assertRunError(t *testing.T, err error, want string) {
} }
} }
func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *RawValidationRegistry { func validatorChainRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *ValidatorChainRegistry {
t.Helper() t.Helper()
registry := NewRawValidationRegistry() registry := NewValidatorChainRegistry()
if err := registry.Register(stage, module, validators...); err != nil { bindings := make([]ModuleBinding, 0, len(validators))
t.Fatalf("register raw validators: %v", err) for _, validator := range validators {
bindings = append(bindings, Binding(validator.Name()))
}
if err := registry.Register(ValidatorChainMapping{Stage: stage, Module: module, Validators: bindings}); err != nil {
t.Fatalf("register validator chain: %v", err)
} }
return registry return registry
} }

View File

@@ -0,0 +1,102 @@
package pipeline
import (
"fmt"
"strings"
)
type validatorChainKey struct {
stage ModuleStage
module string
}
type ValidatorChainMapping struct {
Stage ModuleStage `json:"stage"`
Module string `json:"module"`
Validators []ModuleBinding `json:"validators,omitempty"`
}
type ValidatorChainRegistry struct {
chains map[validatorChainKey][]ModuleBinding
}
func NewValidatorChainRegistry() *ValidatorChainRegistry {
return &ValidatorChainRegistry{
chains: make(map[validatorChainKey][]ModuleBinding),
}
}
func (r *ValidatorChainRegistry) Register(mapping ValidatorChainMapping) error {
if r == nil {
return fmt.Errorf("validator chain registry must not be nil")
}
normalized, err := normalizeValidatorChainMapping(mapping)
if err != nil {
return err
}
if r.chains == nil {
r.chains = make(map[validatorChainKey][]ModuleBinding)
}
key := validatorChainKey{stage: normalized.Stage, module: normalized.Module}
if _, exists := r.chains[key]; exists {
return fmt.Errorf("validator chain for %q %q is already registered", normalized.Stage, normalized.Module)
}
r.chains[key] = cloneModuleBindings(normalized.Validators)
return nil
}
func (r *ValidatorChainRegistry) Validators(stage ModuleStage, module string) []ModuleBinding {
if r == nil {
return nil
}
chain := r.chains[validatorChainKey{stage: stage, module: strings.TrimSpace(module)}]
return cloneModuleBindings(chain)
}
func normalizeValidatorChainMapping(mapping ValidatorChainMapping) (ValidatorChainMapping, error) {
normalized := ValidatorChainMapping{
Stage: mapping.Stage,
Module: strings.TrimSpace(mapping.Module),
Validators: cloneModuleBindings(mapping.Validators),
}
switch normalized.Stage {
case StageChunk, StageExtract, StageMerge, StageNormalize:
default:
return ValidatorChainMapping{}, fmt.Errorf("validator chain stage %q is not supported", normalized.Stage)
}
if normalized.Module == "" {
return ValidatorChainMapping{}, fmt.Errorf("validator chain module key must not be empty")
}
for i, validator := range normalized.Validators {
if strings.TrimSpace(validator.Module) == "" {
return ValidatorChainMapping{}, fmt.Errorf("validator chain for %q %q has empty validator key at index %d", normalized.Stage, normalized.Module, i)
}
normalized.Validators[i] = resolveBinding(validator, "")
}
return normalized, nil
}
func cloneModuleBindings(bindings []ModuleBinding) []ModuleBinding {
if len(bindings) == 0 {
return nil
}
out := make([]ModuleBinding, len(bindings))
for i, binding := range bindings {
out[i] = cloneModuleBinding(binding)
}
return out
}
func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
binding.Module = strings.TrimSpace(binding.Module)
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
binding.Options = cloneOptions(binding.Options)
if len(binding.References) > 0 {
references := make(map[string]string, len(binding.References))
for key, value := range binding.References {
references[key] = value
}
binding.References = references
}
return binding
}

View File

@@ -0,0 +1,74 @@
package pipeline
import "testing"
func TestValidatorChainRegistryRegistersAndLooksUpChains(t *testing.T) {
registry := NewValidatorChainRegistry()
err := registry.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: " extractor ",
Validators: []ModuleBinding{{Module: " first "}, {Module: "second"}},
})
if err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
got := registry.Validators(StageExtract, " extractor ")
if len(got) != 2 || got[0].Module != "first" || got[1].Module != "second" {
t.Fatalf("Validators() = %#v, want trimmed chain", got)
}
}
func TestValidatorChainRegistryRejectsDuplicateMappings(t *testing.T) {
registry := NewValidatorChainRegistry()
mapping := ValidatorChainMapping{Stage: StageMerge, Module: "merge", Validators: []ModuleBinding{Binding("validator")}}
if err := registry.Register(mapping); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
if err := registry.Register(mapping); err == nil {
t.Fatal("Register() error = nil, want duplicate mapping error")
}
}
func TestValidatorChainRegistryRejectsUnsupportedStage(t *testing.T) {
registry := NewValidatorChainRegistry()
err := registry.Register(ValidatorChainMapping{Stage: StageInput, Module: "input"})
if err == nil {
t.Fatal("Register() error = nil, want unsupported stage error")
}
}
func TestValidatorChainRegistryAllowsAbsentAndEmptyChains(t *testing.T) {
registry := NewValidatorChainRegistry()
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
t.Fatalf("absent chain = %#v, want nil", got)
}
if err := registry.Register(ValidatorChainMapping{Stage: StageNormalize, Module: "normalize"}); err != nil {
t.Fatalf("Register(empty) error = %v, want nil", err)
}
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
t.Fatalf("empty chain = %#v, want nil", got)
}
}
func TestValidatorChainRegistryReturnsDefensiveCopies(t *testing.T) {
registry := NewValidatorChainRegistry()
err := registry.Register(ValidatorChainMapping{
Stage: StageChunk,
Module: "chunk",
Validators: []ModuleBinding{{Module: "validator", Options: map[string]any{"level": "strict"}}},
})
if err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
got := registry.Validators(StageChunk, "chunk")
got[0].Module = "changed"
got[0].Options["level"] = "changed"
again := registry.Validators(StageChunk, "chunk")
if again[0].Module != "validator" || again[0].Options["level"] != "strict" {
t.Fatalf("Validators() after caller mutation = %#v, want original chain", again)
}
}

View File

@@ -2,6 +2,7 @@ package pipeline
import ( import (
"fmt" "fmt"
"sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -9,29 +10,34 @@ import (
type ValidatorConstructor func() (contracts.Validator, error) type ValidatorConstructor func() (contracts.Validator, error)
type ValidatorSpec struct {
Key string `json:"key"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
}
type ValidatorRegistry struct { type ValidatorRegistry struct {
constructors map[string]ValidatorConstructor constructors map[string]ValidatorConstructor
specs map[string]ModuleSpec specs map[string]ValidatorSpec
} }
func NewValidatorRegistry() *ValidatorRegistry { func NewValidatorRegistry() *ValidatorRegistry {
return &ValidatorRegistry{ return &ValidatorRegistry{
constructors: make(map[string]ValidatorConstructor), constructors: make(map[string]ValidatorConstructor),
specs: make(map[string]ModuleSpec), specs: make(map[string]ValidatorSpec),
} }
} }
func (r *ValidatorRegistry) Register(key string, constructor ValidatorConstructor) error { func (r *ValidatorRegistry) Register(key string, constructor ValidatorConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageValidate), constructor) return r.RegisterWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
} }
func (r *ValidatorRegistry) RegisterWithSpec(spec ModuleSpec, constructor ValidatorConstructor) error { func (r *ValidatorRegistry) RegisterWithSpec(spec ValidatorSpec, constructor ValidatorConstructor) error {
if r == nil { if r == nil {
return fmt.Errorf("validator registry must not be nil") return fmt.Errorf("validator registry must not be nil")
} }
normalizedSpec := normalizeModuleSpec(spec) normalizedSpec, err := normalizeValidatorSpec(spec)
if err := validateModuleSpec("validator", StageValidate, normalizedSpec); err != nil { if err != nil {
return err return err
} }
if constructor == nil { if constructor == nil {
@@ -45,10 +51,10 @@ func (r *ValidatorRegistry) RegisterWithSpec(spec ModuleSpec, constructor Valida
r.constructors = make(map[string]ValidatorConstructor) r.constructors = make(map[string]ValidatorConstructor)
} }
if r.specs == nil { if r.specs == nil {
r.specs = make(map[string]ModuleSpec) r.specs = make(map[string]ValidatorSpec)
} }
r.constructors[normalizedSpec.Key] = constructor r.constructors[normalizedSpec.Key] = constructor
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec) r.specs[normalizedSpec.Key] = normalizedSpec
return nil return nil
} }
@@ -77,25 +83,45 @@ func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) {
if validator.Name() != normalizedKey { if validator.Name() != normalizedKey {
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name()) return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
} }
switch validator.ExecutionClass() { spec, ok := r.specs[normalizedKey]
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked: if !ok {
default: return nil, fmt.Errorf("validator %q spec is not registered", normalizedKey)
return nil, fmt.Errorf("validator %q returned unsupported execution class %q", normalizedKey, validator.ExecutionClass()) }
if validator.ExecutionClass() != spec.ExecutionClass {
return nil, fmt.Errorf("validator %q returned execution class %q, want %q", normalizedKey, validator.ExecutionClass(), spec.ExecutionClass)
} }
return validator, nil return validator, nil
} }
func (r *ValidatorRegistry) Spec(key string) (ModuleSpec, bool) { func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
if r == nil { if r == nil {
return ModuleSpec{}, false return ValidatorSpec{}, false
} }
spec, ok := r.specs[strings.TrimSpace(key)] spec, ok := r.specs[strings.TrimSpace(key)]
if !ok { if !ok {
return ModuleSpec{}, false return ValidatorSpec{}, false
} }
return cloneModuleSpec(spec), true return spec, true
}
func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec {
if r == nil || len(r.specs) == 0 {
return nil
}
keys := make([]string, 0, len(r.specs))
for key := range r.specs {
keys = append(keys, key)
}
sort.Strings(keys)
specs := make([]ValidatorSpec, 0, len(keys))
for _, key := range keys {
specs = append(specs, r.specs[key])
}
return specs
} }
func (r *ValidatorRegistry) RegisteredKeys() []string { func (r *ValidatorRegistry) RegisteredKeys() []string {
@@ -105,3 +131,19 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
return sortedRegistryKeys(r.constructors) return sortedRegistryKeys(r.constructors)
} }
func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
normalized := ValidatorSpec{
Key: strings.TrimSpace(spec.Key),
ExecutionClass: spec.ExecutionClass,
}
if normalized.Key == "" {
return ValidatorSpec{}, fmt.Errorf("validator key must not be empty")
}
switch normalized.ExecutionClass {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return ValidatorSpec{}, fmt.Errorf("validator %q execution class %q is not supported", normalized.Key, normalized.ExecutionClass)
}
return normalized, nil
}

View File

@@ -2,88 +2,116 @@ package pipeline
import ( import (
"context" "context"
"reflect"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
func TestValidatorRegistryBehavior(t *testing.T) { func TestValidatorRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.Validator]{ registry := NewValidatorRegistry()
name: "ValidatorRegistry", if err := registry.Register(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
key: "generic-validator", t.Fatalf("Register() error = %v, want nil", err)
stage: StageValidate, }
wrongStage: StageExtract,
newRegistry: func() any { validator, err := registry.Build("generic-validator")
return NewValidatorRegistry() if err != nil {
}, t.Fatalf("Build() error = %v, want nil", err)
register: func(registry any, key string, constructor func() (contracts.Validator, error)) error { }
return registry.(*ValidatorRegistry).Register(key, constructor) if validator.Name() != "generic-validator" {
}, t.Fatalf("validator name = %q, want generic-validator", validator.Name())
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.Validator, error)) error { }
return registry.(*ValidatorRegistry).RegisterWithSpec(spec, constructor)
}, spec, ok := registry.Spec(" generic-validator ")
build: func(registry any, key string) (contracts.Validator, error) { if !ok {
return registry.(*ValidatorRegistry).Build(key) t.Fatal("Spec() ok = false, want true")
}, }
spec: func(registry any, key string) (ModuleSpec, bool) { want := ValidatorSpec{Key: "generic-validator", ExecutionClass: contracts.ExecutionClassDeterministic}
return registry.(*ValidatorRegistry).Spec(key) if !reflect.DeepEqual(spec, want) {
}, t.Fatalf("Spec() = %#v, want %#v", spec, want)
registeredKeys: func(registry any) []string { }
return registry.(*ValidatorRegistry).RegisteredKeys() }
},
nilRegister: func(key string, constructor func() (contracts.Validator, error)) error { func TestValidatorRegistryRegistersSpecs(t *testing.T) {
var registry *ValidatorRegistry registry := NewValidatorRegistry()
return registry.Register(key, constructor) spec := ValidatorSpec{Key: " llm-validator ", ExecutionClass: contracts.ExecutionClassLLMBacked}
}, if err := registry.RegisterWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
nilBuild: func(key string) (contracts.Validator, error) { t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
var registry *ValidatorRegistry }
return registry.Build(key)
}, got, ok := registry.Spec("llm-validator")
nilSpec: func(key string) (ModuleSpec, bool) { if !ok {
var registry *ValidatorRegistry t.Fatal("Spec() ok = false, want true")
return registry.Spec(key) }
}, want := ValidatorSpec{Key: "llm-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}
nilRegisteredKey: func() []string { if !reflect.DeepEqual(got, want) {
var registry *ValidatorRegistry t.Fatalf("Spec() = %#v, want %#v", got, want)
return registry.RegisteredKeys() }
}, }
constructor: func(key string) func() (contracts.Validator, error) {
return func() (contracts.Validator, error) { func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
return registryValidator{name: key}, nil registry := NewValidatorRegistry()
} for _, key := range []string{"zeta", "alpha"} {
}, if err := registry.Register(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
moduleKey: func(module contracts.Validator) string { t.Fatalf("Register(%q) error = %v", key, err)
return module.Name() }
}, }
})
specs := registry.RegisteredSpecs()
if len(specs) != 2 || specs[0].Key != "alpha" || specs[1].Key != "zeta" {
t.Fatalf("RegisteredSpecs() = %#v, want sorted specs", specs)
}
} }
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) { func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
registry := NewValidatorRegistry() registry := NewValidatorRegistry()
if err := registry.Register("invalid-validator", func() (contracts.Validator, error) { err := registry.RegisterWithSpec(
return invalidExecutionClassValidator{name: "invalid-validator"}, nil ValidatorSpec{Key: "invalid-validator", ExecutionClass: contracts.ExecutionClass("unsupported")},
}); err != nil { validatorConstructor("invalid-validator", contracts.ExecutionClass("unsupported")),
t.Fatalf("Register() error = %v, want nil", err) )
}
_, err := registry.Build("invalid-validator")
if err == nil { if err == nil {
t.Fatal("Build() error = nil, want unsupported execution class error") t.Fatal("RegisterWithSpec() error = nil, want unsupported execution class error")
} }
} }
type invalidExecutionClassValidator struct { func TestValidatorRegistryRejectsConstructorExecutionClassMismatch(t *testing.T) {
name string registry := NewValidatorRegistry()
if err := registry.RegisterWithSpec(
ValidatorSpec{Key: "validator", ExecutionClass: contracts.ExecutionClassDeterministic},
validatorConstructor("validator", contracts.ExecutionClassLLMBacked),
); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
_, err := registry.Build("validator")
if err == nil {
t.Fatal("Build() error = nil, want execution class mismatch")
}
if !strings.Contains(err.Error(), "execution class") {
t.Fatalf("Build() error = %q, want execution class context", err.Error())
}
} }
func (validator invalidExecutionClassValidator) Name() string { type testValidator struct {
name string
executionClass contracts.ExecutionClass
}
func validatorConstructor(name string, executionClass contracts.ExecutionClass) ValidatorConstructor {
return func() (contracts.Validator, error) {
return testValidator{name: name, executionClass: executionClass}, nil
}
}
func (validator testValidator) Name() string {
return validator.name return validator.name
} }
func (validator invalidExecutionClassValidator) ExecutionClass() contracts.ExecutionClass { func (validator testValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClass("unsupported") return validator.executionClass
} }
func (validator invalidExecutionClassValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { func (validator testValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil return contracts.ValidationResult{Approved: true}, nil
} }

View File

@@ -97,12 +97,13 @@ func walkingSkeletonCatalog(t *testing.T) ModuleCatalog {
t.Helper() t.Helper()
catalog := ModuleCatalog{ catalog := ModuleCatalog{
Inputs: NewInputAdapterRegistry(), Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(), Chunkers: NewChunkerRegistry(),
Extractors: NewExtractorRegistry(), Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(), Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(), Normalizers: NewNormalizerRegistry(),
Outputs: NewOutputEncoderRegistry(), ValidatorChains: NewValidatorChainRegistry(),
Outputs: NewOutputEncoderRegistry(),
} }
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{ if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
Key: "fake/input", Key: "fake/input",

View File

@@ -212,12 +212,13 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
} }
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
Outputs: outputs, ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
} }
} }

View File

@@ -151,12 +151,13 @@ func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.M
}) })
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
Outputs: outputs, ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
} }
} }