eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a framework mechanism for prepared modules and validators to contribute checkpoint identity fingerprints

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-20 15:14:41 -05:00
parent 87c57681f6
commit 0f25e7339f
15 changed files with 539 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/cli/run.go
View File

@@ -352,7 +352,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
@@ -434,6 +434,7 @@ func checkpointHandlersForRun(
settings config.CheckpointCacheConfig,
opts Options,
resolved pipeline.ResolvedPipeline,
componentFingerprints []pipeline.CheckpointFingerprint,
rawInput []byte,
only []string,
llmProfiles []artifacts.LLMProfileManifest,
@@ -454,7 +455,7 @@ func checkpointHandlersForRun(
SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID),
References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: llmProfileFingerprints(llmProfiles),
ProvenanceFingerprints: append(llmProfileFingerprints(llmProfiles), checkpointIdentityFingerprints(componentFingerprints)...),
})
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
@@ -480,6 +481,17 @@ func checkpointHandlersForRun(
return recorder, loader, nil
}
func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint {
if len(values) == 0 {
return nil
}
out := make([]checkpoint.Fingerprint, len(values))
for index, value := range values {
out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value}
}
return out
}
func rawInputDigest(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])

93
internal/cli/spell_catalog_identity_contract_test.go
View File

@@ -11,6 +11,8 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
@@ -110,6 +112,97 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing
}
}
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
if err != nil {
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
if len(fingerprints) != 2 || fingerprints[0].Value != fingerprints[1].Value {
t.Fatalf("prepared fingerprints = %#v, want matching extractor and validator catalog identities", fingerprints)
}
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: materialized,
InputKey: materialized.Input.Module,
RawInputDigest: "sha256:unchanged-input",
References: pipeline.ReferenceProvenance(materialized),
ProvenanceFingerprints: checkpointIdentityFingerprints(values),
})
if identityErr != nil {
t.Fatal(identityErr)
}
return identity
}
first := identityFor(fingerprints)
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
changed[0].Value = "sha256:changed-effective-catalog"
second := identityFor(changed)
if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
}
}
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
if err != nil {
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false)
if err != nil {
t.Fatal(err)
}
doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
doc.Digest, err = source.DigestDocument(&doc)
if err != nil {
t.Fatal(err)
}
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
t.Fatal(err)
}
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true)
if err != nil {
t.Fatal(err)
}
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
}
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
changed[0].Value = "sha256:changed-effective-catalog"
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true)
if err != nil {
t.Fatal(err)
}
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
}
}
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}