Add validator chain provenance
This commit is contained in:
@@ -82,6 +82,14 @@ sanitizing the lane ID:
|
|||||||
"normalizer": "noop"
|
"normalizer": "noop"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
"validator_chains": [
|
||||||
|
{
|
||||||
|
"stage": "extract",
|
||||||
|
"lane_id": "spells",
|
||||||
|
"module_key": "dnd/spells",
|
||||||
|
"validators": []
|
||||||
|
}
|
||||||
|
],
|
||||||
"validation_status": "approved",
|
"validation_status": "approved",
|
||||||
"started_at": "2026-01-01T00:00:00Z",
|
"started_at": "2026-01-01T00:00:00Z",
|
||||||
"completed_at": "2026-01-01T00:00:01Z"
|
"completed_at": "2026-01-01T00:00:01Z"
|
||||||
@@ -103,6 +111,11 @@ references.
|
|||||||
`validation_status` is `approved` when no raw outputs were rejected and
|
`validation_status` is `approved` when no raw outputs were rejected and
|
||||||
`rejected` when one or more raw outputs were rejected.
|
`rejected` when one or more raw outputs were rejected.
|
||||||
|
|
||||||
|
`validator_chains` records the resolved validator chain for each validation
|
||||||
|
point. Entries include stage, lane ID when applicable, module key, and validators
|
||||||
|
with key and execution class. Empty chains are recorded with an empty
|
||||||
|
`validators` array.
|
||||||
|
|
||||||
`normalized_outputs` summarizes each normalized lane output without embedding
|
`normalized_outputs` summarizes each normalized lane output without embedding
|
||||||
payload bytes. Entries include lane ID, normalizer module key, source ID, media
|
payload bytes. Entries include lane ID, normalizer module key, source ID, media
|
||||||
type, and response schema provenance where available.
|
type, and response schema provenance where available.
|
||||||
|
|||||||
@@ -3,17 +3,15 @@
|
|||||||
This roadmap defines the target state for making validation a first-class,
|
This roadmap defines the target state for making validation a first-class,
|
||||||
composable pipeline concern.
|
composable pipeline concern.
|
||||||
|
|
||||||
Current pipeline behavior is raw-output based. The runner can execute
|
Current pipeline behavior is raw-output based. The runner validates `chunk`,
|
||||||
`contracts.RawValidator` chains from `pipeline.RawValidationRegistry` for
|
`extract`, `merge`, and `normalize` outputs through `contracts.Validator`
|
||||||
`chunk`, `extract`, `merge`, and `normalize` outputs. Empty chains approve by
|
chains resolved from `pipeline.ValidatorChainRegistry`. Empty chains approve by
|
||||||
default, validator rejection records a rejected raw output, and rejected output
|
default, validator rejection records a rejected raw output, and rejected output
|
||||||
does not pass to the next stage. Production currently registers no raw
|
does not pass to the next stage. Production currently registers no validators,
|
||||||
validators, and non-empty pipeline-configured validator lists are rejected so
|
and non-empty pipeline-configured validator lists are rejected until
|
||||||
they cannot appear in manifests without executing.
|
stage-scoped override syntax is implemented.
|
||||||
|
|
||||||
Legacy candidate validator contracts and D&D spell validators still exist under
|
The desired end state is that validator implementations, validator
|
||||||
`internal/modules/extract/dnd/spells`, but they are not part of the current
|
|
||||||
runner path. The desired end state is that validator implementations, validator
|
|
||||||
registration, and default module-to-validator mappings are explicit, reviewable,
|
registration, and default module-to-validator mappings are explicit, reviewable,
|
||||||
and independent of concrete module packages.
|
and independent of concrete module packages.
|
||||||
|
|
||||||
@@ -21,9 +19,8 @@ and independent of concrete module packages.
|
|||||||
|
|
||||||
- Move artifact and module-output validation behavior out of `internal/modules`
|
- Move artifact and module-output validation behavior out of `internal/modules`
|
||||||
and into `internal/validators`.
|
and into `internal/validators`.
|
||||||
- Retire or replace the legacy candidate-oriented `contracts.Validator`,
|
- Keep the raw module-output `contracts.Validator`, `ValidationRequest`, and
|
||||||
`ValidationRequest`, and `ValidationResult` path after equivalent raw-output
|
`ValidationResult` path as the single framework validator contract.
|
||||||
validators exist.
|
|
||||||
- Keep each validator in its own package.
|
- Keep each validator in its own package.
|
||||||
- Mirror the stage and domain shape of `internal/modules` where a validator is
|
- Mirror the stage and domain shape of `internal/modules` where a validator is
|
||||||
module-specific.
|
module-specific.
|
||||||
@@ -261,11 +258,10 @@ The validator framework should support validation of outputs from `chunk`,
|
|||||||
enough for stage-specific validators to inspect the output they care about while
|
enough for stage-specific validators to inspect the output they care about while
|
||||||
ignoring irrelevant fields.
|
ignoring irrelevant fields.
|
||||||
|
|
||||||
The current `contracts.RawValidationRequest` is the right starting point. It
|
The current `contracts.ValidationRequest` carries stage, lane, module, source,
|
||||||
already carries stage, lane, module, source, source and chunk provenance,
|
source and chunk provenance, response schema metadata, raw payload, and run
|
||||||
response schema metadata, raw payload, and run metadata. The final contract
|
metadata. The final contract should continue evolving from that raw-output
|
||||||
should evolve from that raw-output shape rather than from the legacy
|
shape.
|
||||||
artifact-candidate `ValidationRequest`.
|
|
||||||
|
|
||||||
Additional fields needed for the full validator system include:
|
Additional fields needed for the full validator system include:
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ func productionRegistries() (pipeline.Registries, error) {
|
|||||||
Mergers: pipeline.NewMergerRegistry(),
|
Mergers: pipeline.NewMergerRegistry(),
|
||||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||||
Validators: pipeline.NewValidatorRegistry(),
|
Validators: pipeline.NewValidatorRegistry(),
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||||
}
|
}
|
||||||
if err := seriatim.Register(registries.Inputs); err != nil {
|
if err := seriatim.Register(registries.Inputs); err != nil {
|
||||||
@@ -99,6 +100,7 @@ func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalo
|
|||||||
Mergers: registries.Mergers,
|
Mergers: registries.Mergers,
|
||||||
Normalizers: registries.Normalizers,
|
Normalizers: registries.Normalizers,
|
||||||
Validators: registries.Validators,
|
Validators: registries.Validators,
|
||||||
|
ValidatorChains: registries.ValidatorChains,
|
||||||
Outputs: registries.Outputs,
|
Outputs: registries.Outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -111,6 +113,7 @@ func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
|
|||||||
Mergers: catalog.Mergers,
|
Mergers: catalog.Mergers,
|
||||||
Normalizers: catalog.Normalizers,
|
Normalizers: catalog.Normalizers,
|
||||||
Validators: catalog.Validators,
|
Validators: catalog.Validators,
|
||||||
|
ValidatorChains: catalog.ValidatorChains,
|
||||||
Outputs: catalog.Outputs,
|
Outputs: catalog.Outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -122,6 +125,7 @@ func isEmptyCatalog(catalog pipeline.ModuleCatalog) bool {
|
|||||||
catalog.Mergers == nil &&
|
catalog.Mergers == nil &&
|
||||||
catalog.Normalizers == nil &&
|
catalog.Normalizers == nil &&
|
||||||
catalog.Validators == nil &&
|
catalog.Validators == nil &&
|
||||||
|
catalog.ValidatorChains == nil &&
|
||||||
catalog.Outputs == nil
|
catalog.Outputs == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -132,6 +136,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
|
|||||||
registries.Mergers == nil &&
|
registries.Mergers == nil &&
|
||||||
registries.Normalizers == nil &&
|
registries.Normalizers == nil &&
|
||||||
registries.Validators == nil &&
|
registries.Validators == nil &&
|
||||||
|
registries.ValidatorChains == nil &&
|
||||||
registries.Outputs == nil
|
registries.Outputs == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2937,6 +2937,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
|||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
Validators: validators,
|
Validators: validators,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,10 +38,21 @@ type ArtifactLaneManifest struct {
|
|||||||
Extractor string `json:"extractor"`
|
Extractor string `json:"extractor"`
|
||||||
Merger string `json:"merger"`
|
Merger string `json:"merger"`
|
||||||
Normalizer string `json:"normalizer"`
|
Normalizer string `json:"normalizer"`
|
||||||
Validators []string `json:"validators,omitempty"`
|
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ValidatorChainManifest struct {
|
||||||
|
Stage string `json:"stage"`
|
||||||
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
|
ModuleKey string `json:"module_key"`
|
||||||
|
Validators []ValidatorManifest `json:"validators"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorManifest struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
ExecutionClass string `json:"execution_class"`
|
||||||
|
}
|
||||||
|
|
||||||
type LLMProfileManifest struct {
|
type LLMProfileManifest struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Provider string `json:"provider,omitempty"`
|
Provider string `json:"provider,omitempty"`
|
||||||
@@ -100,6 +111,7 @@ type RunManifest struct {
|
|||||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||||
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
||||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||||
|
ValidatorChains []ValidatorChainManifest `json:"validator_chains,omitempty"`
|
||||||
References []ReferenceProvenance `json:"references,omitempty"`
|
References []ReferenceProvenance `json:"references,omitempty"`
|
||||||
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
||||||
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
||||||
|
|||||||
@@ -136,12 +136,21 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
Extractor: "event-extractor",
|
Extractor: "event-extractor",
|
||||||
Merger: "appendorder",
|
Merger: "appendorder",
|
||||||
Normalizer: "noop",
|
Normalizer: "noop",
|
||||||
Validators: []string{"grounded"},
|
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
"extractor": map[string]any{"prompt_id": "test.prompt"},
|
"extractor": map[string]any{"prompt_id": "test.prompt"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
ValidatorChains: []ValidatorChainManifest{
|
||||||
|
{
|
||||||
|
Stage: "extract",
|
||||||
|
LaneID: "events",
|
||||||
|
ModuleKey: "event-extractor",
|
||||||
|
Validators: []ValidatorManifest{
|
||||||
|
{Key: "grounded", ExecutionClass: "deterministic"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
gotJSON, err := json.Marshal(manifest)
|
gotJSON, err := json.Marshal(manifest)
|
||||||
@@ -154,7 +163,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles")
|
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "validator_chains", "llm_profiles")
|
||||||
|
|
||||||
profiles, ok := got["llm_profiles"].([]any)
|
profiles, ok := got["llm_profiles"].([]any)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -180,7 +189,20 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
|
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
|
||||||
}
|
}
|
||||||
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
|
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "metadata")
|
||||||
|
|
||||||
|
chains, ok := got["validator_chains"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("validator_chains = %#v, want array", got["validator_chains"])
|
||||||
|
}
|
||||||
|
if len(chains) != 1 {
|
||||||
|
t.Fatalf("len(validator_chains) = %d, want 1", len(chains))
|
||||||
|
}
|
||||||
|
chain, ok := chains[0].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("validator_chains[0] = %#v, want object", chains[0])
|
||||||
|
}
|
||||||
|
assertHasKeys(t, chain, "stage", "lane_id", "module_key", "validators")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
|
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
|
||||||
|
|||||||
@@ -435,6 +435,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
|||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
Validators: validators,
|
Validators: validators,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -476,7 +477,8 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry,
|
|||||||
|
|
||||||
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
|
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
|
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||||
|
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
|
||||||
t.Fatalf("register validator: %v", err)
|
t.Fatalf("register validator: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,6 +97,7 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
|
|||||||
Extractors: extractors,
|
Extractors: extractors,
|
||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,6 +83,18 @@ type ResolvedArtifactLane struct {
|
|||||||
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ResolvedValidatorChain struct {
|
||||||
|
Stage ModuleStage `json:"stage"`
|
||||||
|
LaneID string `json:"lane_id,omitempty"`
|
||||||
|
ModuleKey string `json:"module_key"`
|
||||||
|
Validators []ResolvedValidator `json:"validators"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ResolvedValidator struct {
|
||||||
|
Binding ModuleBinding `json:"binding"`
|
||||||
|
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||||
|
}
|
||||||
|
|
||||||
type ResolvedPipeline struct {
|
type ResolvedPipeline struct {
|
||||||
ID string
|
ID string
|
||||||
Digest string
|
Digest string
|
||||||
@@ -90,6 +102,7 @@ type ResolvedPipeline struct {
|
|||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||||
ArtifactLanes []ResolvedArtifactLane
|
ArtifactLanes []ResolvedArtifactLane
|
||||||
|
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||||
Output ModuleBinding
|
Output ModuleBinding
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,6 +113,7 @@ type ModuleCatalog struct {
|
|||||||
Mergers *MergerRegistry
|
Mergers *MergerRegistry
|
||||||
Normalizers *NormalizerRegistry
|
Normalizers *NormalizerRegistry
|
||||||
Validators *ValidatorRegistry
|
Validators *ValidatorRegistry
|
||||||
|
ValidatorChains *ValidatorChainRegistry
|
||||||
Outputs *OutputEncoderRegistry
|
Outputs *OutputEncoderRegistry
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,15 +186,21 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
|||||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||||
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
||||||
}
|
}
|
||||||
|
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, catalog)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedPipeline{}, err
|
||||||
|
}
|
||||||
|
resolved.ValidatorChains = append(resolved.ValidatorChains, chunkValidatorChain)
|
||||||
outputCapabilities := capabilities.clone()
|
outputCapabilities := capabilities.clone()
|
||||||
|
|
||||||
for _, laneID := range selectedLaneIDs {
|
for _, laneID := range selectedLaneIDs {
|
||||||
laneProfile := lanesByID[laneID]
|
laneProfile := lanesByID[laneID]
|
||||||
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
|
lane, validatorChains, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedPipeline{}, err
|
return ResolvedPipeline{}, err
|
||||||
}
|
}
|
||||||
resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
|
resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
|
||||||
|
resolved.ValidatorChains = append(resolved.ValidatorChains, validatorChains...)
|
||||||
outputCapabilities.addSet(laneCapabilities)
|
outputCapabilities.addSet(laneCapabilities)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,7 +228,7 @@ func resolveArtifactLane(
|
|||||||
options ResolveOptions,
|
options ResolveOptions,
|
||||||
inherited capabilitySet,
|
inherited capabilitySet,
|
||||||
catalog ModuleCatalog,
|
catalog ModuleCatalog,
|
||||||
) (ResolvedArtifactLane, capabilitySet, error) {
|
) (ResolvedArtifactLane, []ResolvedValidatorChain, capabilitySet, error) {
|
||||||
lane := ResolvedArtifactLane{
|
lane := ResolvedArtifactLane{
|
||||||
ID: laneID,
|
ID: laneID,
|
||||||
Extract: resolveBinding(profile.Extract, ""),
|
Extract: resolveBinding(profile.Extract, ""),
|
||||||
@@ -217,17 +237,17 @@ func resolveArtifactLane(
|
|||||||
Validators: resolveBindings(profile.Validators, ""),
|
Validators: resolveBindings(profile.Validators, ""),
|
||||||
}
|
}
|
||||||
if lane.Extract.Module == "" {
|
if lane.Extract.Module == "" {
|
||||||
return ResolvedArtifactLane{}, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
|
return ResolvedArtifactLane{}, nil, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
|
||||||
}
|
}
|
||||||
|
|
||||||
capabilities := inherited.clone()
|
capabilities := inherited.clone()
|
||||||
|
|
||||||
extractSpec, err := extractorSpec(catalog, lane.Extract.Module)
|
extractSpec, err := extractorSpec(catalog, lane.Extract.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
|
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
|
||||||
}
|
}
|
||||||
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
||||||
}
|
}
|
||||||
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
|
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
|
||||||
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
@@ -241,17 +261,17 @@ func resolveArtifactLane(
|
|||||||
Options: options,
|
Options: options,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
||||||
capabilities.add(extractSpec.Provides...)
|
capabilities.add(extractSpec.Provides...)
|
||||||
|
|
||||||
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
|
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
|
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
|
||||||
}
|
}
|
||||||
if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
|
if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
|
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
|
||||||
}
|
}
|
||||||
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
PipelineID: pipelineID,
|
PipelineID: pipelineID,
|
||||||
@@ -264,17 +284,17 @@ func resolveArtifactLane(
|
|||||||
Options: options,
|
Options: options,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
||||||
capabilities.add(mergeSpec.Provides...)
|
capabilities.add(mergeSpec.Provides...)
|
||||||
|
|
||||||
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
|
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
|
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
|
||||||
}
|
}
|
||||||
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
|
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
|
||||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
|
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
|
||||||
}
|
}
|
||||||
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||||
PipelineID: pipelineID,
|
PipelineID: pipelineID,
|
||||||
@@ -287,22 +307,105 @@ func resolveArtifactLane(
|
|||||||
Options: options,
|
Options: options,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
||||||
capabilities.add(normalizeSpec.Provides...)
|
capabilities.add(normalizeSpec.Provides...)
|
||||||
|
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
return ResolvedArtifactLane{}, nil, configuredValidatorsError(pipelineID, laneID)
|
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
|
||||||
}
|
}
|
||||||
|
|
||||||
return lane, capabilities, nil
|
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, catalog)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
|
}
|
||||||
|
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, catalog)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
|
}
|
||||||
|
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, catalog)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
|
}
|
||||||
|
validatorChains := []ResolvedValidatorChain{extractValidatorChain, mergeValidatorChain, normalizeValidatorChain}
|
||||||
|
|
||||||
|
return lane, validatorChains, capabilities, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func configuredValidatorsError(pipelineID string, laneID string) error {
|
func configuredValidatorsError(pipelineID string, laneID string) error {
|
||||||
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID)
|
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
|
||||||
|
chain := ResolvedValidatorChain{
|
||||||
|
Stage: stage,
|
||||||
|
LaneID: strings.TrimSpace(laneID),
|
||||||
|
ModuleKey: strings.TrimSpace(module),
|
||||||
|
}
|
||||||
|
if chain.ModuleKey == "" {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain %q module key must not be empty", pipelineID, stage)
|
||||||
|
}
|
||||||
|
switch stage {
|
||||||
|
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
||||||
|
default:
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
|
||||||
|
}
|
||||||
|
|
||||||
|
if catalog.ValidatorChains == nil {
|
||||||
|
return chain, nil
|
||||||
|
}
|
||||||
|
bindings := catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
|
||||||
|
if len(bindings) == 0 {
|
||||||
|
return chain, nil
|
||||||
|
}
|
||||||
|
if catalog.Validators == nil {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator registry must not be nil for %s validator chain on module %q", pipelineID, stage, chain.ModuleKey)
|
||||||
|
}
|
||||||
|
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
|
||||||
|
for _, validator := range bindings {
|
||||||
|
spec, ok := catalog.Validators.Spec(validator.Module)
|
||||||
|
if !ok {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
|
||||||
|
}
|
||||||
|
chain.Validators = append(chain.Validators, ResolvedValidator{
|
||||||
|
Binding: cloneModuleBinding(validator),
|
||||||
|
ExecutionClass: spec.ExecutionClass,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return chain, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneResolvedValidatorChains(chains []ResolvedValidatorChain) []ResolvedValidatorChain {
|
||||||
|
if len(chains) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]ResolvedValidatorChain, len(chains))
|
||||||
|
for i, chain := range chains {
|
||||||
|
out[i] = ResolvedValidatorChain{
|
||||||
|
Stage: chain.Stage,
|
||||||
|
LaneID: strings.TrimSpace(chain.LaneID),
|
||||||
|
ModuleKey: strings.TrimSpace(chain.ModuleKey),
|
||||||
|
Validators: cloneResolvedValidators(chain.Validators),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator {
|
||||||
|
if len(validators) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]ResolvedValidator, len(validators))
|
||||||
|
for i, validator := range validators {
|
||||||
|
out[i] = ResolvedValidator{
|
||||||
|
Binding: cloneModuleBinding(validator.Binding),
|
||||||
|
ExecutionClass: validator.ExecutionClass,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
|
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
|
||||||
return ResolvedReferenceTarget{
|
return ResolvedReferenceTarget{
|
||||||
Stage: stage,
|
Stage: stage,
|
||||||
|
|||||||
@@ -110,6 +110,78 @@ func TestResolvePipelineAppliesDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: []ModuleBinding{Binding("grounded")},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {Extract: Binding("event-extractor")},
|
||||||
|
},
|
||||||
|
}, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(resolved.ValidatorChains) != 4 {
|
||||||
|
t.Fatalf("len(ValidatorChains) = %d, want chunk plus lane extract/merge/normalize", len(resolved.ValidatorChains))
|
||||||
|
}
|
||||||
|
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||||
|
if extractChain == nil {
|
||||||
|
t.Fatal("extract validator chain not found")
|
||||||
|
}
|
||||||
|
if len(extractChain.Validators) != 1 {
|
||||||
|
t.Fatalf("extract validators = %#v, want one validator", extractChain.Validators)
|
||||||
|
}
|
||||||
|
if extractChain.Validators[0].Binding.Module != "grounded" {
|
||||||
|
t.Fatalf("extract validator key = %q, want grounded", extractChain.Validators[0].Binding.Module)
|
||||||
|
}
|
||||||
|
if extractChain.Validators[0].ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("extract validator execution class = %q, want deterministic", extractChain.Validators[0].ExecutionClass)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", DefaultChunkModule)
|
||||||
|
if chunkChain == nil {
|
||||||
|
t.Fatal("chunk validator chain not found")
|
||||||
|
}
|
||||||
|
if len(chunkChain.Validators) != 0 {
|
||||||
|
t.Fatalf("chunk validators = %#v, want explicit empty chain", chunkChain.Validators)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageNormalize,
|
||||||
|
Module: DefaultNormalizeModule,
|
||||||
|
Validators: []ModuleBinding{Binding("missing-validator")},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ResolvePipeline(PipelineProfile{
|
||||||
|
ID: "invalid-chain",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {Extract: Binding("event-extractor")},
|
||||||
|
},
|
||||||
|
}, ResolveOptions{}, catalog)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "missing-validator") {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
|
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
|
||||||
profile := multiLaneProfile()
|
profile := multiLaneProfile()
|
||||||
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
|
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
|
||||||
@@ -955,6 +1027,15 @@ func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName str
|
|||||||
t.Fatalf("binding %q not found in %#v", slotName, bindings)
|
t.Fatalf("binding %q not found in %#v", slotName, bindings)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func findResolvedValidatorChain(chains []ResolvedValidatorChain, stage ModuleStage, laneID string, module string) *ResolvedValidatorChain {
|
||||||
|
for i := range chains {
|
||||||
|
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
|
||||||
|
return &chains[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func newProfileCatalog(t *testing.T) ModuleCatalog {
|
func newProfileCatalog(t *testing.T) ModuleCatalog {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1000,6 +1081,7 @@ func emptyProfileCatalog() ModuleCatalog {
|
|||||||
Mergers: NewMergerRegistry(),
|
Mergers: NewMergerRegistry(),
|
||||||
Normalizers: NewNormalizerRegistry(),
|
Normalizers: NewNormalizerRegistry(),
|
||||||
Validators: NewValidatorRegistry(),
|
Validators: NewValidatorRegistry(),
|
||||||
|
ValidatorChains: NewValidatorChainRegistry(),
|
||||||
Outputs: NewOutputEncoderRegistry(),
|
Outputs: NewOutputEncoderRegistry(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1043,7 +1125,8 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
|
|||||||
t.Fatalf("register normalizer spec %#v: %v", spec, err)
|
t.Fatalf("register normalizer spec %#v: %v", spec, err)
|
||||||
}
|
}
|
||||||
case StageValidate:
|
case StageValidate:
|
||||||
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
|
validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||||
|
if err := catalog.Validators.RegisterWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||||
t.Fatalf("register validator spec %#v: %v", spec, err)
|
t.Fatalf("register validator spec %#v: %v", spec, err)
|
||||||
}
|
}
|
||||||
case StageOutput:
|
case StageOutput:
|
||||||
|
|||||||
@@ -1,78 +0,0 @@
|
|||||||
package pipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
)
|
|
||||||
|
|
||||||
type rawValidationKey struct {
|
|
||||||
stage ModuleStage
|
|
||||||
module string
|
|
||||||
}
|
|
||||||
|
|
||||||
type RawValidationRegistry struct {
|
|
||||||
chains map[rawValidationKey][]contracts.Validator
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRawValidationRegistry() *RawValidationRegistry {
|
|
||||||
return &RawValidationRegistry{
|
|
||||||
chains: make(map[rawValidationKey][]contracts.Validator),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.Validator) error {
|
|
||||||
if r == nil {
|
|
||||||
return fmt.Errorf("raw validation registry must not be nil")
|
|
||||||
}
|
|
||||||
normalizedModule := strings.TrimSpace(module)
|
|
||||||
if normalizedModule == "" {
|
|
||||||
return fmt.Errorf("raw validation module key must not be empty")
|
|
||||||
}
|
|
||||||
switch stage {
|
|
||||||
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("raw validation stage %q is not supported", stage)
|
|
||||||
}
|
|
||||||
if len(validators) == 0 {
|
|
||||||
return fmt.Errorf("raw validation chain for %q %q must not be empty", stage, normalizedModule)
|
|
||||||
}
|
|
||||||
|
|
||||||
chain := make([]contracts.Validator, 0, len(validators))
|
|
||||||
for i, validator := range validators {
|
|
||||||
if validator == nil {
|
|
||||||
return fmt.Errorf("raw validator %d for %q %q must not be nil", i, stage, normalizedModule)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(validator.Name()) == "" {
|
|
||||||
return fmt.Errorf("raw validator %d for %q %q must not have an empty name", i, stage, normalizedModule)
|
|
||||||
}
|
|
||||||
switch validator.ExecutionClass() {
|
|
||||||
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("raw validator %q for %q %q has unsupported execution class %q", validator.Name(), stage, normalizedModule, validator.ExecutionClass())
|
|
||||||
}
|
|
||||||
chain = append(chain, validator)
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.chains == nil {
|
|
||||||
r.chains = make(map[rawValidationKey][]contracts.Validator)
|
|
||||||
}
|
|
||||||
key := rawValidationKey{stage: stage, module: normalizedModule}
|
|
||||||
if _, exists := r.chains[key]; exists {
|
|
||||||
return fmt.Errorf("raw validation chain for %q %q is already registered", stage, normalizedModule)
|
|
||||||
}
|
|
||||||
r.chains[key] = append([]contracts.Validator(nil), chain...)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.Validator {
|
|
||||||
if r == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
chain := r.chains[rawValidationKey{stage: stage, module: strings.TrimSpace(module)}]
|
|
||||||
if len(chain) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return append([]contracts.Validator(nil), chain...)
|
|
||||||
}
|
|
||||||
@@ -24,7 +24,7 @@ type Registries struct {
|
|||||||
Mergers *MergerRegistry
|
Mergers *MergerRegistry
|
||||||
Normalizers *NormalizerRegistry
|
Normalizers *NormalizerRegistry
|
||||||
Validators *ValidatorRegistry
|
Validators *ValidatorRegistry
|
||||||
RawValidators *RawValidationRegistry
|
ValidatorChains *ValidatorChainRegistry
|
||||||
Outputs *OutputEncoderRegistry
|
Outputs *OutputEncoderRegistry
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -127,7 +127,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
||||||
}
|
}
|
||||||
rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, input.Metadata, attempt)
|
rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, input.Metadata, input.Pipeline.ValidatorChains, attempt)
|
||||||
if err != nil || rejection != nil {
|
if err != nil || rejection != nil {
|
||||||
return false, rejection, err
|
return false, rejection, err
|
||||||
}
|
}
|
||||||
@@ -240,6 +240,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
schema: extractOutput.Schema,
|
schema: extractOutput.Schema,
|
||||||
payload: extractOutput.Payload,
|
payload: extractOutput.Payload,
|
||||||
metadata: input.Metadata,
|
metadata: input.Metadata,
|
||||||
|
chains: input.Pipeline.ValidatorChains,
|
||||||
attempt: attempt,
|
attempt: attempt,
|
||||||
})
|
})
|
||||||
if err != nil || rejection != nil {
|
if err != nil || rejection != nil {
|
||||||
@@ -296,6 +297,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
schema: mergeOutput.Schema,
|
schema: mergeOutput.Schema,
|
||||||
payload: mergeOutput.Payload,
|
payload: mergeOutput.Payload,
|
||||||
metadata: input.Metadata,
|
metadata: input.Metadata,
|
||||||
|
chains: input.Pipeline.ValidatorChains,
|
||||||
attempt: attempt,
|
attempt: attempt,
|
||||||
})
|
})
|
||||||
if err != nil || rejection != nil {
|
if err != nil || rejection != nil {
|
||||||
@@ -346,6 +348,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
|||||||
schema: normalizeOutput.Schema,
|
schema: normalizeOutput.Schema,
|
||||||
payload: normalizeOutput.Payload,
|
payload: normalizeOutput.Payload,
|
||||||
metadata: input.Metadata,
|
metadata: input.Metadata,
|
||||||
|
chains: input.Pipeline.ValidatorChains,
|
||||||
attempt: attempt,
|
attempt: attempt,
|
||||||
})
|
})
|
||||||
if err != nil || rejection != nil {
|
if err != nil || rejection != nil {
|
||||||
@@ -378,6 +381,7 @@ type rawValidationTarget struct {
|
|||||||
schema contracts.ResponseSchema
|
schema contracts.ResponseSchema
|
||||||
payload contracts.RawPayload
|
payload contracts.RawPayload
|
||||||
metadata map[string]any
|
metadata map[string]any
|
||||||
|
chains []ResolvedValidatorChain
|
||||||
attempt int
|
attempt int
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -428,7 +432,7 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
|
|||||||
return false, lastRejection, nil
|
return false, lastRejection, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, metadata map[string]any, attempt int) (*contracts.RejectedOutput, error) {
|
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, metadata map[string]any, chains []ResolvedValidatorChain, attempt int) (*contracts.RejectedOutput, error) {
|
||||||
for _, chunk := range chunks {
|
for _, chunk := range chunks {
|
||||||
_, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
_, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||||
stage: StageChunk,
|
stage: StageChunk,
|
||||||
@@ -443,6 +447,7 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
|
|||||||
Metadata: cloneMetadata(chunk.Metadata),
|
Metadata: cloneMetadata(chunk.Metadata),
|
||||||
},
|
},
|
||||||
metadata: metadata,
|
metadata: metadata,
|
||||||
|
chains: chains,
|
||||||
attempt: attempt,
|
attempt: attempt,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -456,10 +461,16 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||||
validators := r.registries.RawValidators.Validators(target.stage, target.moduleKey)
|
chain := resolvedValidatorChain(target.stage, target.laneID, target.moduleKey, target.chains)
|
||||||
if len(validators) == 0 {
|
if len(chain.Validators) == 0 {
|
||||||
|
chain = r.registryValidatorChain(target.stage, target.laneID, target.moduleKey)
|
||||||
|
}
|
||||||
|
if len(chain.Validators) == 0 {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
|
if r.registries.Validators == nil {
|
||||||
|
return nil, nil, fmt.Errorf("validator registry must not be nil")
|
||||||
|
}
|
||||||
|
|
||||||
request := contracts.ValidationRequest{
|
request := contracts.ValidationRequest{
|
||||||
Stage: string(target.stage),
|
Stage: string(target.stage),
|
||||||
@@ -475,7 +486,11 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
|
|||||||
}
|
}
|
||||||
|
|
||||||
var warnings []contracts.Warning
|
var warnings []contracts.Warning
|
||||||
for _, validator := range validators {
|
for _, validatorBinding := range chain.Validators {
|
||||||
|
validator, err := r.registries.Validators.Build(validatorBinding.Binding.Module)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
|
||||||
|
}
|
||||||
result, err := validator.Validate(ctx, request)
|
result, err := validator.Validate(ctx, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
|
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
|
||||||
@@ -507,6 +522,60 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
|
|||||||
return warnings, nil, nil
|
return warnings, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Runner) registryValidatorChain(stage ModuleStage, laneID string, moduleKey string) ResolvedValidatorChain {
|
||||||
|
chain := ResolvedValidatorChain{
|
||||||
|
Stage: stage,
|
||||||
|
LaneID: strings.TrimSpace(laneID),
|
||||||
|
ModuleKey: strings.TrimSpace(moduleKey),
|
||||||
|
}
|
||||||
|
if r == nil || r.registries.ValidatorChains == nil {
|
||||||
|
return chain
|
||||||
|
}
|
||||||
|
bindings := r.registries.ValidatorChains.Validators(stage, moduleKey)
|
||||||
|
if len(bindings) == 0 {
|
||||||
|
return chain
|
||||||
|
}
|
||||||
|
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
|
||||||
|
for _, binding := range bindings {
|
||||||
|
executionClass := contracts.ExecutionClass("")
|
||||||
|
if r.registries.Validators != nil {
|
||||||
|
if spec, ok := r.registries.Validators.Spec(binding.Module); ok {
|
||||||
|
executionClass = spec.ExecutionClass
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chain.Validators = append(chain.Validators, ResolvedValidator{
|
||||||
|
Binding: cloneModuleBinding(binding),
|
||||||
|
ExecutionClass: executionClass,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return chain
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
|
||||||
|
for _, chain := range chains {
|
||||||
|
if chain.Stage != stage {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if chain.ModuleKey != moduleKey {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(chain.LaneID) != strings.TrimSpace(laneID) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return ResolvedValidatorChain{
|
||||||
|
Stage: chain.Stage,
|
||||||
|
LaneID: chain.LaneID,
|
||||||
|
ModuleKey: chain.ModuleKey,
|
||||||
|
Validators: cloneResolvedValidators(chain.Validators),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ResolvedValidatorChain{
|
||||||
|
Stage: stage,
|
||||||
|
LaneID: strings.TrimSpace(laneID),
|
||||||
|
ModuleKey: strings.TrimSpace(moduleKey),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) validateRegistries(pipeline ResolvedPipeline) error {
|
func (r *Runner) validateRegistries(pipeline ResolvedPipeline) error {
|
||||||
if r.registries.Inputs == nil {
|
if r.registries.Inputs == nil {
|
||||||
return fmt.Errorf("input registry must not be nil")
|
return fmt.Errorf("input registry must not be nil")
|
||||||
@@ -586,6 +655,7 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
|
|||||||
Chunker: pipeline.Chunk.Module,
|
Chunker: pipeline.Chunk.Module,
|
||||||
OutputEncoder: pipeline.Output.Module,
|
OutputEncoder: pipeline.Output.Module,
|
||||||
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
|
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
|
||||||
|
ValidatorChains: validatorChainManifests(pipeline.ValidatorChains),
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
StartedAt: timePtr(startedAt),
|
StartedAt: timePtr(startedAt),
|
||||||
References: ReferenceProvenance(pipeline),
|
References: ReferenceProvenance(pipeline),
|
||||||
@@ -607,6 +677,29 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
|
|||||||
return manifest
|
return manifest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validatorChainManifests(chains []ResolvedValidatorChain) []artifacts.ValidatorChainManifest {
|
||||||
|
if len(chains) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
manifests := make([]artifacts.ValidatorChainManifest, 0, len(chains))
|
||||||
|
for _, chain := range chains {
|
||||||
|
manifest := artifacts.ValidatorChainManifest{
|
||||||
|
Stage: string(chain.Stage),
|
||||||
|
LaneID: chain.LaneID,
|
||||||
|
ModuleKey: chain.ModuleKey,
|
||||||
|
Validators: make([]artifacts.ValidatorManifest, 0, len(chain.Validators)),
|
||||||
|
}
|
||||||
|
for _, validator := range chain.Validators {
|
||||||
|
manifest.Validators = append(manifest.Validators, artifacts.ValidatorManifest{
|
||||||
|
Key: validator.Binding.Module,
|
||||||
|
ExecutionClass: string(validator.ExecutionClass),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
manifests = append(manifests, manifest)
|
||||||
|
}
|
||||||
|
return manifests
|
||||||
|
}
|
||||||
|
|
||||||
func failOutput(output RunOutput) RunOutput {
|
func failOutput(output RunOutput) RunOutput {
|
||||||
if output.Manifest.PipelineID != "" {
|
if output.Manifest.PipelineID != "" {
|
||||||
populateRawOutputManifest(&output)
|
populateRawOutputManifest(&output)
|
||||||
|
|||||||
@@ -887,8 +887,9 @@ func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
|
func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"}
|
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -909,8 +910,9 @@ func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
|
func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
|
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -933,8 +935,9 @@ func TestRunOmitsLaneWithNoAcceptedExtractOutputs(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
|
func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-merge", approved: []bool{false}, reason: "bad_merge", message: "merge rejected"}
|
validator := &runnerChainValidator{name: "chain-merge", approved: []bool{false}, reason: "bad_merge", message: "merge rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageMerge, "merge", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
modules.validatorChains = validatorChainRegistry(t, StageMerge, "merge", validator)
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -954,8 +957,9 @@ func TestRunRejectedMergePreventsNormalizeForLane(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunRejectedNormalizePreventsOutputForLane(t *testing.T) {
|
func TestRunRejectedNormalizePreventsOutputForLane(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-normalize", approved: []bool{false}, reason: "bad_normalize", message: "normalize rejected"}
|
validator := &runnerChainValidator{name: "chain-normalize", approved: []bool{false}, reason: "bad_normalize", message: "normalize rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageNormalize, "normalize", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
modules.validatorChains = validatorChainRegistry(t, StageNormalize, "normalize", validator)
|
||||||
|
|
||||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -999,8 +1003,9 @@ func TestRunRetriesSameModuleInputAfterFrameworkError(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
|
func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false, true, true}, reason: "bad_extract", message: "extract rejected"}
|
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true, true}, reason: "bad_extract", message: "extract rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
|
||||||
pipeline := resolvedPipeline()
|
pipeline := resolvedPipeline()
|
||||||
pipeline.ArtifactLanes[0].Extract.Retries = 1
|
pipeline.ArtifactLanes[0].Extract.Retries = 1
|
||||||
|
|
||||||
@@ -1023,8 +1028,9 @@ func TestRunRetriesSameModuleInputAfterValidatorRejection(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
|
func TestRunStopsRetryAfterConfiguredAttemptsAndRecordsAttemptCount(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
validator := &runnerRawValidator{name: "raw-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
|
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false}, reason: "bad_extract", message: "extract rejected"}
|
||||||
modules.rawValidators = rawValidationRegistry(t, StageExtract, "extract-alpha", validator)
|
modules.validators[validator.name] = validator
|
||||||
|
modules.validatorChains = validatorChainRegistry(t, StageExtract, "extract-alpha", validator)
|
||||||
pipeline := resolvedPipeline()
|
pipeline := resolvedPipeline()
|
||||||
pipeline.ArtifactLanes[0].Extract.Retries = 1
|
pipeline.ArtifactLanes[0].Extract.Retries = 1
|
||||||
|
|
||||||
@@ -1312,8 +1318,11 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
|
|||||||
if lane.ID != "alpha" || lane.Extractor != "extract-alpha" || lane.Merger != "merge" || lane.Normalizer != "normalize" {
|
if lane.ID != "alpha" || lane.Extractor != "extract-alpha" || lane.Merger != "merge" || lane.Normalizer != "normalize" {
|
||||||
t.Fatalf("ArtifactLanes[0] = %#v, want lane details", lane)
|
t.Fatalf("ArtifactLanes[0] = %#v, want lane details", lane)
|
||||||
}
|
}
|
||||||
if len(lane.Validators) != 0 {
|
if len(manifest.ValidatorChains) != 4 {
|
||||||
t.Fatalf("lane validators = %#v, want none", lane.Validators)
|
t.Fatalf("ValidatorChains = %#v, want four validation points", manifest.ValidatorChains)
|
||||||
|
}
|
||||||
|
if manifest.ValidatorChains[0].Stage != string(StageChunk) || manifest.ValidatorChains[0].ModuleKey != "chunk" || len(manifest.ValidatorChains[0].Validators) != 0 {
|
||||||
|
t.Fatalf("chunk validator chain = %#v, want explicit empty chunk chain", manifest.ValidatorChains[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1454,6 +1463,12 @@ func resolvedPipeline() ResolvedPipeline {
|
|||||||
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
|
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
ValidatorChains: []ResolvedValidatorChain{
|
||||||
|
{Stage: StageChunk, ModuleKey: "chunk"},
|
||||||
|
{Stage: StageExtract, LaneID: "alpha", ModuleKey: "extract-alpha"},
|
||||||
|
{Stage: StageMerge, LaneID: "alpha", ModuleKey: "merge"},
|
||||||
|
{Stage: StageNormalize, LaneID: "alpha", ModuleKey: "normalize"},
|
||||||
|
},
|
||||||
Output: Binding("output"),
|
Output: Binding("output"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1493,8 +1508,8 @@ type runnerModules struct {
|
|||||||
extractors map[string]*runnerExtractor
|
extractors map[string]*runnerExtractor
|
||||||
mergers map[string]*runnerMerger
|
mergers map[string]*runnerMerger
|
||||||
normalizers map[string]*runnerNormalizer
|
normalizers map[string]*runnerNormalizer
|
||||||
validators map[string]*runnerValidator
|
validators map[string]contracts.Validator
|
||||||
rawValidators *RawValidationRegistry
|
validatorChains *ValidatorChainRegistry
|
||||||
output *runnerOutputEncoder
|
output *runnerOutputEncoder
|
||||||
inputBuildErr error
|
inputBuildErr error
|
||||||
chunkerBuildErr error
|
chunkerBuildErr error
|
||||||
@@ -1513,9 +1528,9 @@ func defaultRunnerModules() *runnerModules {
|
|||||||
normalizers: map[string]*runnerNormalizer{
|
normalizers: map[string]*runnerNormalizer{
|
||||||
"normalize": {key: "normalize"},
|
"normalize": {key: "normalize"},
|
||||||
},
|
},
|
||||||
validators: map[string]*runnerValidator{
|
validators: map[string]contracts.Validator{
|
||||||
"configured": {name: "configured"},
|
"configured": &runnerValidator{name: "configured"},
|
||||||
"second-validator": {name: "second-validator"},
|
"second-validator": &runnerValidator{name: "second-validator"},
|
||||||
},
|
},
|
||||||
output: &runnerOutputEncoder{
|
output: &runnerOutputEncoder{
|
||||||
key: "output",
|
key: "output",
|
||||||
@@ -1539,7 +1554,7 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
|
|||||||
Mergers: NewMergerRegistry(),
|
Mergers: NewMergerRegistry(),
|
||||||
Normalizers: NewNormalizerRegistry(),
|
Normalizers: NewNormalizerRegistry(),
|
||||||
Validators: NewValidatorRegistry(),
|
Validators: NewValidatorRegistry(),
|
||||||
RawValidators: modules.rawValidators,
|
ValidatorChains: modules.validatorChains,
|
||||||
Outputs: NewOutputEncoderRegistry(),
|
Outputs: NewOutputEncoderRegistry(),
|
||||||
}
|
}
|
||||||
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
|
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
|
||||||
@@ -1807,7 +1822,7 @@ type runnerValidator struct {
|
|||||||
requests []contracts.ValidationRequest
|
requests []contracts.ValidationRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
type runnerRawValidator struct {
|
type runnerChainValidator struct {
|
||||||
name string
|
name string
|
||||||
approved []bool
|
approved []bool
|
||||||
reason string
|
reason string
|
||||||
@@ -1818,15 +1833,15 @@ type runnerRawValidator struct {
|
|||||||
requests []contracts.ValidationRequest
|
requests []contracts.ValidationRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator *runnerRawValidator) Name() string {
|
func (validator *runnerChainValidator) Name() string {
|
||||||
return validator.name
|
return validator.name
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator *runnerRawValidator) ExecutionClass() contracts.ExecutionClass {
|
func (validator *runnerChainValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
return contracts.ExecutionClassDeterministic
|
return contracts.ExecutionClassDeterministic
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator *runnerRawValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
func (validator *runnerChainValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
validator.calls++
|
validator.calls++
|
||||||
validator.requests = append(validator.requests, req)
|
validator.requests = append(validator.requests, req)
|
||||||
if validator.err != nil {
|
if validator.err != nil {
|
||||||
@@ -2037,12 +2052,16 @@ func assertRunError(t *testing.T, err error, want string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *RawValidationRegistry {
|
func validatorChainRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *ValidatorChainRegistry {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
registry := NewRawValidationRegistry()
|
registry := NewValidatorChainRegistry()
|
||||||
if err := registry.Register(stage, module, validators...); err != nil {
|
bindings := make([]ModuleBinding, 0, len(validators))
|
||||||
t.Fatalf("register raw validators: %v", err)
|
for _, validator := range validators {
|
||||||
|
bindings = append(bindings, Binding(validator.Name()))
|
||||||
|
}
|
||||||
|
if err := registry.Register(ValidatorChainMapping{Stage: stage, Module: module, Validators: bindings}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
}
|
}
|
||||||
return registry
|
return registry
|
||||||
}
|
}
|
||||||
|
|||||||
102
internal/framework/pipeline/validator_chain_registry.go
Normal file
102
internal/framework/pipeline/validator_chain_registry.go
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type validatorChainKey struct {
|
||||||
|
stage ModuleStage
|
||||||
|
module string
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorChainMapping struct {
|
||||||
|
Stage ModuleStage `json:"stage"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Validators []ModuleBinding `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorChainRegistry struct {
|
||||||
|
chains map[validatorChainKey][]ModuleBinding
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewValidatorChainRegistry() *ValidatorChainRegistry {
|
||||||
|
return &ValidatorChainRegistry{
|
||||||
|
chains: make(map[validatorChainKey][]ModuleBinding),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ValidatorChainRegistry) Register(mapping ValidatorChainMapping) error {
|
||||||
|
if r == nil {
|
||||||
|
return fmt.Errorf("validator chain registry must not be nil")
|
||||||
|
}
|
||||||
|
normalized, err := normalizeValidatorChainMapping(mapping)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if r.chains == nil {
|
||||||
|
r.chains = make(map[validatorChainKey][]ModuleBinding)
|
||||||
|
}
|
||||||
|
key := validatorChainKey{stage: normalized.Stage, module: normalized.Module}
|
||||||
|
if _, exists := r.chains[key]; exists {
|
||||||
|
return fmt.Errorf("validator chain for %q %q is already registered", normalized.Stage, normalized.Module)
|
||||||
|
}
|
||||||
|
r.chains[key] = cloneModuleBindings(normalized.Validators)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ValidatorChainRegistry) Validators(stage ModuleStage, module string) []ModuleBinding {
|
||||||
|
if r == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
chain := r.chains[validatorChainKey{stage: stage, module: strings.TrimSpace(module)}]
|
||||||
|
return cloneModuleBindings(chain)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeValidatorChainMapping(mapping ValidatorChainMapping) (ValidatorChainMapping, error) {
|
||||||
|
normalized := ValidatorChainMapping{
|
||||||
|
Stage: mapping.Stage,
|
||||||
|
Module: strings.TrimSpace(mapping.Module),
|
||||||
|
Validators: cloneModuleBindings(mapping.Validators),
|
||||||
|
}
|
||||||
|
switch normalized.Stage {
|
||||||
|
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
||||||
|
default:
|
||||||
|
return ValidatorChainMapping{}, fmt.Errorf("validator chain stage %q is not supported", normalized.Stage)
|
||||||
|
}
|
||||||
|
if normalized.Module == "" {
|
||||||
|
return ValidatorChainMapping{}, fmt.Errorf("validator chain module key must not be empty")
|
||||||
|
}
|
||||||
|
for i, validator := range normalized.Validators {
|
||||||
|
if strings.TrimSpace(validator.Module) == "" {
|
||||||
|
return ValidatorChainMapping{}, fmt.Errorf("validator chain for %q %q has empty validator key at index %d", normalized.Stage, normalized.Module, i)
|
||||||
|
}
|
||||||
|
normalized.Validators[i] = resolveBinding(validator, "")
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneModuleBindings(bindings []ModuleBinding) []ModuleBinding {
|
||||||
|
if len(bindings) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]ModuleBinding, len(bindings))
|
||||||
|
for i, binding := range bindings {
|
||||||
|
out[i] = cloneModuleBinding(binding)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
|
||||||
|
binding.Module = strings.TrimSpace(binding.Module)
|
||||||
|
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
|
||||||
|
binding.Options = cloneOptions(binding.Options)
|
||||||
|
if len(binding.References) > 0 {
|
||||||
|
references := make(map[string]string, len(binding.References))
|
||||||
|
for key, value := range binding.References {
|
||||||
|
references[key] = value
|
||||||
|
}
|
||||||
|
binding.References = references
|
||||||
|
}
|
||||||
|
return binding
|
||||||
|
}
|
||||||
74
internal/framework/pipeline/validator_chain_registry_test.go
Normal file
74
internal/framework/pipeline/validator_chain_registry_test.go
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryRegistersAndLooksUpChains(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
err := registry.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: " extractor ",
|
||||||
|
Validators: []ModuleBinding{{Module: " first "}, {Module: "second"}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := registry.Validators(StageExtract, " extractor ")
|
||||||
|
if len(got) != 2 || got[0].Module != "first" || got[1].Module != "second" {
|
||||||
|
t.Fatalf("Validators() = %#v, want trimmed chain", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryRejectsDuplicateMappings(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
mapping := ValidatorChainMapping{Stage: StageMerge, Module: "merge", Validators: []ModuleBinding{Binding("validator")}}
|
||||||
|
if err := registry.Register(mapping); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := registry.Register(mapping); err == nil {
|
||||||
|
t.Fatal("Register() error = nil, want duplicate mapping error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryRejectsUnsupportedStage(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
err := registry.Register(ValidatorChainMapping{Stage: StageInput, Module: "input"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Register() error = nil, want unsupported stage error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryAllowsAbsentAndEmptyChains(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
|
||||||
|
t.Fatalf("absent chain = %#v, want nil", got)
|
||||||
|
}
|
||||||
|
if err := registry.Register(ValidatorChainMapping{Stage: StageNormalize, Module: "normalize"}); err != nil {
|
||||||
|
t.Fatalf("Register(empty) error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
|
||||||
|
t.Fatalf("empty chain = %#v, want nil", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorChainRegistryReturnsDefensiveCopies(t *testing.T) {
|
||||||
|
registry := NewValidatorChainRegistry()
|
||||||
|
err := registry.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageChunk,
|
||||||
|
Module: "chunk",
|
||||||
|
Validators: []ModuleBinding{{Module: "validator", Options: map[string]any{"level": "strict"}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := registry.Validators(StageChunk, "chunk")
|
||||||
|
got[0].Module = "changed"
|
||||||
|
got[0].Options["level"] = "changed"
|
||||||
|
|
||||||
|
again := registry.Validators(StageChunk, "chunk")
|
||||||
|
if again[0].Module != "validator" || again[0].Options["level"] != "strict" {
|
||||||
|
t.Fatalf("Validators() after caller mutation = %#v, want original chain", again)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
@@ -9,29 +10,34 @@ import (
|
|||||||
|
|
||||||
type ValidatorConstructor func() (contracts.Validator, error)
|
type ValidatorConstructor func() (contracts.Validator, error)
|
||||||
|
|
||||||
|
type ValidatorSpec struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||||
|
}
|
||||||
|
|
||||||
type ValidatorRegistry struct {
|
type ValidatorRegistry struct {
|
||||||
constructors map[string]ValidatorConstructor
|
constructors map[string]ValidatorConstructor
|
||||||
specs map[string]ModuleSpec
|
specs map[string]ValidatorSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewValidatorRegistry() *ValidatorRegistry {
|
func NewValidatorRegistry() *ValidatorRegistry {
|
||||||
return &ValidatorRegistry{
|
return &ValidatorRegistry{
|
||||||
constructors: make(map[string]ValidatorConstructor),
|
constructors: make(map[string]ValidatorConstructor),
|
||||||
specs: make(map[string]ModuleSpec),
|
specs: make(map[string]ValidatorSpec),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ValidatorRegistry) Register(key string, constructor ValidatorConstructor) error {
|
func (r *ValidatorRegistry) Register(key string, constructor ValidatorConstructor) error {
|
||||||
return r.RegisterWithSpec(defaultModuleSpec(key, StageValidate), constructor)
|
return r.RegisterWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ValidatorRegistry) RegisterWithSpec(spec ModuleSpec, constructor ValidatorConstructor) error {
|
func (r *ValidatorRegistry) RegisterWithSpec(spec ValidatorSpec, constructor ValidatorConstructor) error {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return fmt.Errorf("validator registry must not be nil")
|
return fmt.Errorf("validator registry must not be nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
normalizedSpec := normalizeModuleSpec(spec)
|
normalizedSpec, err := normalizeValidatorSpec(spec)
|
||||||
if err := validateModuleSpec("validator", StageValidate, normalizedSpec); err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if constructor == nil {
|
if constructor == nil {
|
||||||
@@ -45,10 +51,10 @@ func (r *ValidatorRegistry) RegisterWithSpec(spec ModuleSpec, constructor Valida
|
|||||||
r.constructors = make(map[string]ValidatorConstructor)
|
r.constructors = make(map[string]ValidatorConstructor)
|
||||||
}
|
}
|
||||||
if r.specs == nil {
|
if r.specs == nil {
|
||||||
r.specs = make(map[string]ModuleSpec)
|
r.specs = make(map[string]ValidatorSpec)
|
||||||
}
|
}
|
||||||
r.constructors[normalizedSpec.Key] = constructor
|
r.constructors[normalizedSpec.Key] = constructor
|
||||||
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
|
r.specs[normalizedSpec.Key] = normalizedSpec
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,25 +83,45 @@ func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) {
|
|||||||
if validator.Name() != normalizedKey {
|
if validator.Name() != normalizedKey {
|
||||||
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
|
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
|
||||||
}
|
}
|
||||||
switch validator.ExecutionClass() {
|
spec, ok := r.specs[normalizedKey]
|
||||||
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
|
if !ok {
|
||||||
default:
|
return nil, fmt.Errorf("validator %q spec is not registered", normalizedKey)
|
||||||
return nil, fmt.Errorf("validator %q returned unsupported execution class %q", normalizedKey, validator.ExecutionClass())
|
}
|
||||||
|
if validator.ExecutionClass() != spec.ExecutionClass {
|
||||||
|
return nil, fmt.Errorf("validator %q returned execution class %q, want %q", normalizedKey, validator.ExecutionClass(), spec.ExecutionClass)
|
||||||
}
|
}
|
||||||
|
|
||||||
return validator, nil
|
return validator, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ValidatorRegistry) Spec(key string) (ModuleSpec, bool) {
|
func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return ModuleSpec{}, false
|
return ValidatorSpec{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
spec, ok := r.specs[strings.TrimSpace(key)]
|
spec, ok := r.specs[strings.TrimSpace(key)]
|
||||||
if !ok {
|
if !ok {
|
||||||
return ModuleSpec{}, false
|
return ValidatorSpec{}, false
|
||||||
}
|
}
|
||||||
return cloneModuleSpec(spec), true
|
return spec, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec {
|
||||||
|
if r == nil || len(r.specs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := make([]string, 0, len(r.specs))
|
||||||
|
for key := range r.specs {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
specs := make([]ValidatorSpec, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
specs = append(specs, r.specs[key])
|
||||||
|
}
|
||||||
|
return specs
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ValidatorRegistry) RegisteredKeys() []string {
|
func (r *ValidatorRegistry) RegisteredKeys() []string {
|
||||||
@@ -105,3 +131,19 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
|
|||||||
|
|
||||||
return sortedRegistryKeys(r.constructors)
|
return sortedRegistryKeys(r.constructors)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
|
||||||
|
normalized := ValidatorSpec{
|
||||||
|
Key: strings.TrimSpace(spec.Key),
|
||||||
|
ExecutionClass: spec.ExecutionClass,
|
||||||
|
}
|
||||||
|
if normalized.Key == "" {
|
||||||
|
return ValidatorSpec{}, fmt.Errorf("validator key must not be empty")
|
||||||
|
}
|
||||||
|
switch normalized.ExecutionClass {
|
||||||
|
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
|
||||||
|
default:
|
||||||
|
return ValidatorSpec{}, fmt.Errorf("validator %q execution class %q is not supported", normalized.Key, normalized.ExecutionClass)
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,88 +2,116 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestValidatorRegistryBehavior(t *testing.T) {
|
func TestValidatorRegistryBehavior(t *testing.T) {
|
||||||
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.Validator]{
|
registry := NewValidatorRegistry()
|
||||||
name: "ValidatorRegistry",
|
if err := registry.Register(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
|
||||||
key: "generic-validator",
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
stage: StageValidate,
|
}
|
||||||
wrongStage: StageExtract,
|
|
||||||
newRegistry: func() any {
|
validator, err := registry.Build("generic-validator")
|
||||||
return NewValidatorRegistry()
|
if err != nil {
|
||||||
},
|
t.Fatalf("Build() error = %v, want nil", err)
|
||||||
register: func(registry any, key string, constructor func() (contracts.Validator, error)) error {
|
}
|
||||||
return registry.(*ValidatorRegistry).Register(key, constructor)
|
if validator.Name() != "generic-validator" {
|
||||||
},
|
t.Fatalf("validator name = %q, want generic-validator", validator.Name())
|
||||||
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.Validator, error)) error {
|
}
|
||||||
return registry.(*ValidatorRegistry).RegisterWithSpec(spec, constructor)
|
|
||||||
},
|
spec, ok := registry.Spec(" generic-validator ")
|
||||||
build: func(registry any, key string) (contracts.Validator, error) {
|
if !ok {
|
||||||
return registry.(*ValidatorRegistry).Build(key)
|
t.Fatal("Spec() ok = false, want true")
|
||||||
},
|
}
|
||||||
spec: func(registry any, key string) (ModuleSpec, bool) {
|
want := ValidatorSpec{Key: "generic-validator", ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||||
return registry.(*ValidatorRegistry).Spec(key)
|
if !reflect.DeepEqual(spec, want) {
|
||||||
},
|
t.Fatalf("Spec() = %#v, want %#v", spec, want)
|
||||||
registeredKeys: func(registry any) []string {
|
}
|
||||||
return registry.(*ValidatorRegistry).RegisteredKeys()
|
}
|
||||||
},
|
|
||||||
nilRegister: func(key string, constructor func() (contracts.Validator, error)) error {
|
func TestValidatorRegistryRegistersSpecs(t *testing.T) {
|
||||||
var registry *ValidatorRegistry
|
registry := NewValidatorRegistry()
|
||||||
return registry.Register(key, constructor)
|
spec := ValidatorSpec{Key: " llm-validator ", ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||||
},
|
if err := registry.RegisterWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
|
||||||
nilBuild: func(key string) (contracts.Validator, error) {
|
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||||
var registry *ValidatorRegistry
|
}
|
||||||
return registry.Build(key)
|
|
||||||
},
|
got, ok := registry.Spec("llm-validator")
|
||||||
nilSpec: func(key string) (ModuleSpec, bool) {
|
if !ok {
|
||||||
var registry *ValidatorRegistry
|
t.Fatal("Spec() ok = false, want true")
|
||||||
return registry.Spec(key)
|
}
|
||||||
},
|
want := ValidatorSpec{Key: "llm-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||||
nilRegisteredKey: func() []string {
|
if !reflect.DeepEqual(got, want) {
|
||||||
var registry *ValidatorRegistry
|
t.Fatalf("Spec() = %#v, want %#v", got, want)
|
||||||
return registry.RegisteredKeys()
|
}
|
||||||
},
|
}
|
||||||
constructor: func(key string) func() (contracts.Validator, error) {
|
|
||||||
return func() (contracts.Validator, error) {
|
func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
|
||||||
return registryValidator{name: key}, nil
|
registry := NewValidatorRegistry()
|
||||||
|
for _, key := range []string{"zeta", "alpha"} {
|
||||||
|
if err := registry.Register(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
|
||||||
|
t.Fatalf("Register(%q) error = %v", key, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
specs := registry.RegisteredSpecs()
|
||||||
|
if len(specs) != 2 || specs[0].Key != "alpha" || specs[1].Key != "zeta" {
|
||||||
|
t.Fatalf("RegisteredSpecs() = %#v, want sorted specs", specs)
|
||||||
}
|
}
|
||||||
},
|
|
||||||
moduleKey: func(module contracts.Validator) string {
|
|
||||||
return module.Name()
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
|
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
|
||||||
registry := NewValidatorRegistry()
|
registry := NewValidatorRegistry()
|
||||||
if err := registry.Register("invalid-validator", func() (contracts.Validator, error) {
|
err := registry.RegisterWithSpec(
|
||||||
return invalidExecutionClassValidator{name: "invalid-validator"}, nil
|
ValidatorSpec{Key: "invalid-validator", ExecutionClass: contracts.ExecutionClass("unsupported")},
|
||||||
}); err != nil {
|
validatorConstructor("invalid-validator", contracts.ExecutionClass("unsupported")),
|
||||||
t.Fatalf("Register() error = %v, want nil", err)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
_, err := registry.Build("invalid-validator")
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Build() error = nil, want unsupported execution class error")
|
t.Fatal("RegisterWithSpec() error = nil, want unsupported execution class error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type invalidExecutionClassValidator struct {
|
func TestValidatorRegistryRejectsConstructorExecutionClassMismatch(t *testing.T) {
|
||||||
|
registry := NewValidatorRegistry()
|
||||||
|
if err := registry.RegisterWithSpec(
|
||||||
|
ValidatorSpec{Key: "validator", ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||||
|
validatorConstructor("validator", contracts.ExecutionClassLLMBacked),
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := registry.Build("validator")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Build() error = nil, want execution class mismatch")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "execution class") {
|
||||||
|
t.Fatalf("Build() error = %q, want execution class context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type testValidator struct {
|
||||||
name string
|
name string
|
||||||
|
executionClass contracts.ExecutionClass
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator invalidExecutionClassValidator) Name() string {
|
func validatorConstructor(name string, executionClass contracts.ExecutionClass) ValidatorConstructor {
|
||||||
|
return func() (contracts.Validator, error) {
|
||||||
|
return testValidator{name: name, executionClass: executionClass}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (validator testValidator) Name() string {
|
||||||
return validator.name
|
return validator.name
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator invalidExecutionClassValidator) ExecutionClass() contracts.ExecutionClass {
|
func (validator testValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
return contracts.ExecutionClass("unsupported")
|
return validator.executionClass
|
||||||
}
|
}
|
||||||
|
|
||||||
func (validator invalidExecutionClassValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
func (validator testValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||||
return contracts.ValidationResult{Approved: true}, nil
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ func walkingSkeletonCatalog(t *testing.T) ModuleCatalog {
|
|||||||
Extractors: NewExtractorRegistry(),
|
Extractors: NewExtractorRegistry(),
|
||||||
Mergers: NewMergerRegistry(),
|
Mergers: NewMergerRegistry(),
|
||||||
Normalizers: NewNormalizerRegistry(),
|
Normalizers: NewNormalizerRegistry(),
|
||||||
|
ValidatorChains: NewValidatorChainRegistry(),
|
||||||
Outputs: NewOutputEncoderRegistry(),
|
Outputs: NewOutputEncoderRegistry(),
|
||||||
}
|
}
|
||||||
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
|
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
|
||||||
|
|||||||
@@ -217,6 +217,7 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
|
|||||||
Extractors: extractors,
|
Extractors: extractors,
|
||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -156,6 +156,7 @@ func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.M
|
|||||||
Extractors: extractors,
|
Extractors: extractors,
|
||||||
Mergers: mergers,
|
Mergers: mergers,
|
||||||
Normalizers: normalizers,
|
Normalizers: normalizers,
|
||||||
|
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user