eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a framework mechanism for prepared modules and validators to contribute checkpoint identity fingerprints

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-20 15:14:41 -05:00
parent 87c57681f6
commit 0f25e7339f
15 changed files with 539 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
internal/cli/spell_catalog_identity_contract_test.go
View File

@@ -11,6 +11,8 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
@@ -110,6 +112,97 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing
}
}
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
if err != nil {
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
if len(fingerprints) != 2 || fingerprints[0].Value != fingerprints[1].Value {
t.Fatalf("prepared fingerprints = %#v, want matching extractor and validator catalog identities", fingerprints)
}
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: materialized,
InputKey: materialized.Input.Module,
RawInputDigest: "sha256:unchanged-input",
References: pipeline.ReferenceProvenance(materialized),
ProvenanceFingerprints: checkpointIdentityFingerprints(values),
})
if identityErr != nil {
t.Fatal(identityErr)
}
return identity
}
first := identityFor(fingerprints)
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
changed[0].Value = "sha256:changed-effective-catalog"
second := identityFor(changed)
if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
}
}
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
if err != nil {
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false)
if err != nil {
t.Fatal(err)
}
doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
doc.Digest, err = source.DigestDocument(&doc)
if err != nil {
t.Fatal(err)
}
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
t.Fatal(err)
}
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true)
if err != nil {
t.Fatal(err)
}
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
}
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
changed[0].Value = "sha256:changed-effective-catalog"
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true)
if err != nil {
t.Fatal(err)
}
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
}
}
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}