eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Include validator policy in pipeline identity

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 13:33:16 +00:00
parent 3e4fa923eb
commit 35e45f0914
4 changed files with 193 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/framework/pipeline/profile.go
View File

@@ -1010,6 +1010,7 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget
ArtifactLanes []ResolvedArtifactLane
ValidatorChains []ResolvedValidatorChain
Output ModuleBinding
}{
ID: resolved.ID,
@@ -1017,6 +1018,7 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
Chunk: resolved.Chunk,
ChunkReferences: resolved.ChunkReferences,
ArtifactLanes: resolved.ArtifactLanes,
ValidatorChains: resolved.ValidatorChains,
Output: resolved.Output,
}
encoded, err := json.Marshal(withoutDigest)

180
internal/framework/pipeline/profile_test.go
View File

@@ -1062,6 +1062,186 @@ func TestResolvePipelineDigestChangesWhenBindingChanges(t *testing.T) {
}
}
func TestResolvePipelineDigestIncludesEffectiveValidatorChain(t *testing.T) {
resolve := func(t *testing.T, validators []ModuleBinding, explicitEmpty bool) ResolvedPipeline {
t.Helper()
catalog := newProfileCatalog(t)
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "second-validator", ExecutionClass: contracts.ExecutionClassDeterministic})
if len(validators) > 0 {
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: validators,
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
}
profile := baselineProfile()
if explicitEmpty {
lane := profile.Artifacts["events"]
lane.Extract.Validators = ValidatorOverride{Set: true}
profile.Artifacts["events"] = lane
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
return resolved
}
inherited := resolve(t, []ModuleBinding{Binding("grounded"), Binding("second-validator")}, false)
repeated := resolve(t, []ModuleBinding{Binding("grounded"), Binding("second-validator")}, false)
if inherited.Digest != repeated.Digest {
t.Fatalf("same resolved validator policy produced digests %q and %q", inherited.Digest, repeated.Digest)
}
tests := []struct {
name string
left ResolvedPipeline
right ResolvedPipeline
}{
{
name: "different default",
left: resolve(t, []ModuleBinding{Binding("grounded")}, false),
right: resolve(t, []ModuleBinding{Binding("second-validator")}, false),
},
{
name: "added validator",
left: resolve(t, nil, false),
right: resolve(t, []ModuleBinding{Binding("grounded")}, false),
},
{
name: "removed validator",
left: resolve(t, []ModuleBinding{Binding("grounded")}, false),
right: resolve(t, nil, false),
},
{
name: "reordered chain",
left: inherited,
right: resolve(t, []ModuleBinding{Binding("second-validator"), Binding("grounded")}, false),
},
{
name: "explicit empty",
left: inherited,
right: resolve(t, []ModuleBinding{Binding("grounded"), Binding("second-validator")}, true),
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if tc.left.Digest == tc.right.Digest {
t.Fatalf("digest = %q for both validator policies, want changed digest", tc.left.Digest)
}
})
}
}
func TestResolvedPipelineDigestIncludesCompleteValidatorPolicy(t *testing.T) {
base := validatorDigestFixture()
baseDigest, err := resolvedPipelineDigest(base)
if err != nil {
t.Fatalf("resolvedPipelineDigest(base) error = %v, want nil", err)
}
tests := []struct {
name string
mutate func(*ResolvedPipeline)
}{
{name: "stage", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Stage = StageMerge }},
{name: "lane", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].LaneID = "other" }},
{name: "owner module", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].ModuleKey = "other-extractor" }},
{name: "validator order", mutate: func(value *ResolvedPipeline) {
value.ValidatorChains[0].Validators[0], value.ValidatorChains[0].Validators[1] = value.ValidatorChains[0].Validators[1], value.ValidatorChains[0].Validators[0]
}},
{name: "validator module", mutate: func(value *ResolvedPipeline) {
value.ValidatorChains[0].Validators[0].Binding.Module = "other-validator"
}},
{name: "llm profile", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.LLMProfile = "fast" }},
{name: "retries", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].Binding.Retries++ }},
{name: "options", mutate: func(value *ResolvedPipeline) {
value.ValidatorChains[0].Validators[0].Binding.Options["threshold"] = 0.75
}},
{name: "references", mutate: func(value *ResolvedPipeline) {
value.ValidatorChains[0].Validators[0].Binding.References["rules"] = "other.md"
}},
{name: "execution class", mutate: func(value *ResolvedPipeline) {
value.ValidatorChains[0].Validators[0].ExecutionClass = contracts.ExecutionClassDeterministic
}},
{name: "target", mutate: func(value *ResolvedPipeline) {
value.ValidatorChains[0].Validators[0].Target = ValidatorTargetSerialized
}},
{name: "artifact kind", mutate: func(value *ResolvedPipeline) { value.ValidatorChains[0].Validators[0].ArtifactKind = "test/other" }},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
changed := validatorDigestFixture()
tc.mutate(&changed)
digest, err := resolvedPipelineDigest(changed)
if err != nil {
t.Fatalf("resolvedPipelineDigest(changed) error = %v, want nil", err)
}
if digest == baseDigest {
t.Fatalf("digest = %q after %s change, want different digest", digest, tc.name)
}
})
}
}
func TestResolvedPipelineDigestCanonicalizesValidatorBindingMaps(t *testing.T) {
left := validatorDigestFixture()
right := validatorDigestFixture()
right.ValidatorChains[0].Validators[0].Binding.Options = map[string]any{}
right.ValidatorChains[0].Validators[0].Binding.Options["threshold"] = 0.5
right.ValidatorChains[0].Validators[0].Binding.Options["mode"] = "strict"
right.ValidatorChains[0].Validators[0].Binding.References = map[string]string{}
right.ValidatorChains[0].Validators[0].Binding.References["examples"] = "examples.md"
right.ValidatorChains[0].Validators[0].Binding.References["rules"] = "rules.md"
leftDigest, err := resolvedPipelineDigest(left)
if err != nil {
t.Fatalf("resolvedPipelineDigest(left) error = %v, want nil", err)
}
rightDigest, err := resolvedPipelineDigest(right)
if err != nil {
t.Fatalf("resolvedPipelineDigest(right) error = %v, want nil", err)
}
if leftDigest != rightDigest {
t.Fatalf("equivalent validator maps produced digests %q and %q", leftDigest, rightDigest)
}
}
func validatorDigestFixture() ResolvedPipeline {
return ResolvedPipeline{
ID: "validator-policy",
ValidatorChains: []ResolvedValidatorChain{{
Stage: StageExtract,
LaneID: "events",
ModuleKey: "event-extractor",
Validators: []ResolvedValidator{
{
Binding: ModuleBinding{
Module: "semantic-validator",
LLMProfile: "careful",
Retries: 2,
Options: map[string]any{"mode": "strict", "threshold": 0.5},
References: map[string]string{"rules": "rules.md", "examples": "examples.md"},
},
ExecutionClass: contracts.ExecutionClassLLMBacked,
Target: ValidatorTargetTyped,
ArtifactKind: "test/notes",
},
{
Binding: Binding("grounded"),
ExecutionClass: contracts.ExecutionClassDeterministic,
Target: ValidatorTargetTyped,
ArtifactKind: "test/notes",
},
},
}},
}
}
func TestBindingTrimsModuleAndLeavesResolutionFieldsEmpty(t *testing.T) {
binding := Binding(" module ")
if binding.Module != "module" {