Add a framework mechanism for prepared modules and validators to contribute checkpoint identity fingerprints

This commit is contained in:
2026-07-20 15:14:41 -05:00
parent 87c57681f6
commit 0f25e7339f
15 changed files with 539 additions and 26 deletions

View File

@@ -158,7 +158,12 @@ reference into an immutable effective catalog and adds a generated
canonical-name-only JSON input to every structured completion request. Overlay
failures therefore stop construction before source parsing or an LLM call;
campaign references remain separate disambiguation inputs and never become
source evidence. The separate `internal/modules/dnd/codec/spells` package
source evidence. Both the extractor and deterministic catalog validator expose
the effective base-plus-overlay semantic digest as scoped prepared-component
checkpoint identity. Raw overlay provenance independently covers file-byte
changes, while the semantic digest also invalidates reuse when the embedded
catalog or catalog composition changes. The separate
`internal/modules/dnd/codec/spells` package
owns the durable schema and stable JSON representation for artifact kind
`dnd/spell-list`. The runner keeps the result typed through validators and later
stages, using the codec only for checkpoint, debug, and output boundaries.

View File

@@ -126,8 +126,16 @@ then output. It stops at the first error with pipeline, stage, lane, module, and
validator context as applicable. It never invokes an operation method.
`PreparedPipeline` keeps private constructed executors and exposes cloned
resolved input, chunk, lane, and output identities. `pipeline.RunInput` carries
that prepared pipeline, raw source input, run identity and timing, optional
resolved input, chunk, lane, and output identities. Prepared components may
implement `pipeline.CheckpointFingerprintProvider` to contribute explicit
semantic identities to checkpoint reuse. Preparation trims and validates each
non-secret name and value, prefixes it with the component's stage, lane,
module, and validator scope, rejects duplicates, and retains the resulting
sorted collection behind a defensive-copy accessor. Fingerprints must be
stable and must not contain source content, credentials, local paths,
timestamps, or other invocation-specific values.
`pipeline.RunInput` carries that prepared pipeline, raw source input, run identity and timing, optional
session and profile metadata, a chunk-plan store and mode, and checkpoint/debug
collaborators. The runner
parses source bytes through the already constructed input adapter. Later stage
@@ -267,6 +275,14 @@ bytes, and decode failures become explicit reuse misses and execute the step
normally. Dependency fingerprints and debug content digests use the same stable
codec bytes that cross those boundaries.
The CLI includes prepared-component fingerprints in the run-wide checkpoint
identity alongside resolved configuration, raw input, reference provenance,
runtime overrides, and LLM-profile fingerprints. Module metadata is not used
implicitly for cache identity: components opt in only with stable semantic
values that can change accepted output. Adding or changing a component
fingerprint intentionally produces a cold cache miss. Existing checkpoint
schemas and paths remain unchanged.
Debug instrumentation wraps run, stage, attempt, validator, and structured LLM
boundaries. Every executed chunk, extract, merge, and normalize attempt writes
one terminal envelope for acceptance, validator rejection, module or validator

View File

@@ -25,6 +25,9 @@ not `bypass`.
`internal/framework/checkpoint` owns checkpoint identity, manifests, payload
codecs, loader, and recorder. The CLI constructs a recorder whenever checkpoint
recording is enabled and constructs a loader only for a `--resume` invocation.
Identity incorporates explicit stable semantic fingerprints collected from
prepared modules and validators in addition to configuration, input,
references, runtime overrides, and LLM profiles.
The serialized
`workspace_schema_version` identifiers are frozen wire-compatibility fields;
they do not describe a current public state surface.

View File

@@ -89,6 +89,11 @@ Checkpoints use the selected root and the existing identity hierarchy:
<checkpoint-root>/<pipeline-id>/<input-key>-<source-or-input-digest>/<pipeline-digest>/<identity-digest>/...
```
The final identity digest includes stable semantic fingerprints explicitly
contributed by prepared modules and validators. Adding or changing one of
these fingerprints intentionally causes a cold cache miss; old checkpoint
directories are left in place and are never migrated or deleted automatically.
An empty configured directory selects
`<os.UserCacheDir>/notarius/checkpoints`. The root is exact when configured.
Created directories and files use `0700` and `0600` permissions on supported

View File

@@ -6,7 +6,10 @@ The accepted baseline implementation is complete. It provides effective
SRD-plus-overlay catalog composition, canonical-name prompt grounding,
deterministic catalog validation, the maintained `retries: 2` policy, and
provenance and checkpoint identity coverage through the assembled offline
workflow.
workflow. Checkpoint identity includes independently scoped extractor and
validator fingerprints of the effective catalog, closing the earlier gap in
which raw overlay changes invalidated reuse but embedded catalog changes did
not.
External quality evaluation is pending. This repository contains the
maintained example and offline fake-LLM coverage, but no approved

View File

@@ -352,7 +352,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
@@ -434,6 +434,7 @@ func checkpointHandlersForRun(
settings config.CheckpointCacheConfig,
opts Options,
resolved pipeline.ResolvedPipeline,
componentFingerprints []pipeline.CheckpointFingerprint,
rawInput []byte,
only []string,
llmProfiles []artifacts.LLMProfileManifest,
@@ -454,7 +455,7 @@ func checkpointHandlersForRun(
SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID),
References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: llmProfileFingerprints(llmProfiles),
ProvenanceFingerprints: append(llmProfileFingerprints(llmProfiles), checkpointIdentityFingerprints(componentFingerprints)...),
})
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
@@ -480,6 +481,17 @@ func checkpointHandlersForRun(
return recorder, loader, nil
}
func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint {
if len(values) == 0 {
return nil
}
out := make([]checkpoint.Fingerprint, len(values))
for index, value := range values {
out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value}
}
return out
}
func rawInputDigest(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])

View File

@@ -11,6 +11,8 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
@@ -110,6 +112,97 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing
}
}
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
if err != nil {
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
if len(fingerprints) != 2 || fingerprints[0].Value != fingerprints[1].Value {
t.Fatalf("prepared fingerprints = %#v, want matching extractor and validator catalog identities", fingerprints)
}
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: materialized,
InputKey: materialized.Input.Module,
RawInputDigest: "sha256:unchanged-input",
References: pipeline.ReferenceProvenance(materialized),
ProvenanceFingerprints: checkpointIdentityFingerprints(values),
})
if identityErr != nil {
t.Fatal(identityErr)
}
return identity
}
first := identityFor(fingerprints)
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
changed[0].Value = "sha256:changed-effective-catalog"
second := identityFor(changed)
if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
}
}
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
if err != nil {
t.Fatal(err)
}
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
if err != nil {
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false)
if err != nil {
t.Fatal(err)
}
doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
doc.Digest, err = source.DigestDocument(&doc)
if err != nil {
t.Fatal(err)
}
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
t.Fatal(err)
}
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true)
if err != nil {
t.Fatal(err)
}
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
}
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
changed[0].Value = "sha256:changed-effective-catalog"
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true)
if err != nil {
t.Fatal(err)
}
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
}
}
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}

View File

@@ -16,6 +16,13 @@ type CheckpointFingerprint struct {
Value string `json:"value"`
}
// CheckpointFingerprintProvider supplies stable, non-secret semantic identity
// for a prepared module or validator. Values must not contain source content,
// credentials, local paths, timestamps, or other invocation-specific data.
type CheckpointFingerprintProvider interface {
CheckpointFingerprints() []CheckpointFingerprint
}
type CheckpointRecorder interface {
SourceRunning(moduleKey string) error
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error

View File

@@ -72,6 +72,167 @@ func TestPrepareConstructsEverythingInStableOrder(t *testing.T) {
}
}
func TestPreparedCheckpointFingerprintsCollectEveryComponentInStableScopeOrder(t *testing.T) {
provider := func(value string) checkpointFingerprintTestProvider {
return checkpointFingerprintTestProvider{{Name: "identity", Value: value}}
}
prepared := &PreparedPipeline{
resolved: ResolvedPipeline{
ID: "fingerprints",
Input: Binding("input"),
Chunk: Binding("chunk"),
Output: Binding("output"),
},
input: provider("input"),
chunker: provider("chunk"),
chunkValidators: preparedValidatorChain{validators: []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("chunk-validator"), Target: ValidatorTargetChunk},
chunk: provider("chunk-validator"),
}}},
output: provider("output"),
}
lane := preparedLaneExecutor{
resolved: ResolvedArtifactLane{
ID: "spells",
Extract: Binding("extract"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
},
typed: &preparedTypedLane{
extractor: provider("extract"),
merger: provider("merge"),
normalizer: provider("normalize"),
},
extractValidators: fingerprintTestValidatorChain("extract-validator", provider("extract-validator")),
mergeValidators: fingerprintTestValidatorChain("merge-validator", provider("merge-validator")),
normalizeValidators: fingerprintTestValidatorChain("normalize-validator", provider("normalize-validator")),
}
prepared.lanes = []preparedLaneExecutor{lane}
first, err := collectPreparedCheckpointFingerprints(prepared)
if err != nil {
t.Fatalf("collectPreparedCheckpointFingerprints() error = %v, want nil", err)
}
second, err := collectPreparedCheckpointFingerprints(prepared)
if err != nil {
t.Fatalf("second collection error = %v, want nil", err)
}
if !reflect.DeepEqual(first, second) {
t.Fatalf("fingerprints are not deterministic: %#v / %#v", first, second)
}
if len(first) != 10 {
t.Fatalf("fingerprints = %#v, want all 10 prepared components", first)
}
for index := 1; index < len(first); index++ {
if first[index-1].Name >= first[index].Name {
t.Fatalf("fingerprints are not sorted: %#v", first)
}
}
prepared.checkpointFingerprints = first
copy := prepared.CheckpointFingerprints()
copy[0].Value = "mutated"
if reflect.DeepEqual(copy, prepared.CheckpointFingerprints()) {
t.Fatal("CheckpointFingerprints() exposed mutable backing storage")
}
}
func TestPreparedCheckpointFingerprintsValidateProviderValues(t *testing.T) {
tests := []struct {
name string
fingerprints []CheckpointFingerprint
want string
}{
{name: "empty name", fingerprints: []CheckpointFingerprint{{Value: "value"}}, want: "non-empty name and value"},
{name: "empty value", fingerprints: []CheckpointFingerprint{{Name: "name"}}, want: "non-empty name and value"},
{name: "duplicate after trimming", fingerprints: []CheckpointFingerprint{{Name: "same", Value: "one"}, {Name: " same ", Value: "two"}}, want: "duplicated"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
prepared := &PreparedPipeline{
resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")},
input: checkpointFingerprintTestProvider(test.fingerprints),
}
_, err := collectPreparedCheckpointFingerprints(prepared)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("error = %v, want substring %q", err, test.want)
}
})
}
}
func TestPreparedCheckpointFingerprintsRemainEmptyWithoutProviders(t *testing.T) {
registries, _ := constructionRegistries(t, nil, nil)
resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog())
if err != nil {
t.Fatal(err)
}
prepared, err := Prepare(resolved, registries, ModuleDependencies{})
if err != nil {
t.Fatal(err)
}
if got := prepared.CheckpointFingerprints(); len(got) != 0 {
t.Fatalf("CheckpointFingerprints() = %#v, want empty", got)
}
}
func TestPreparedCheckpointFingerprintsScopeRepeatedValidatorsByPosition(t *testing.T) {
provider := checkpointFingerprintTestProvider{{Name: "identity", Value: "same"}}
validator := func() preparedValidator {
return preparedValidator{resolved: ResolvedValidator{Binding: Binding("configured"), Target: ValidatorTargetChunk}, chunk: provider}
}
prepared := &PreparedPipeline{
resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")},
chunkValidators: preparedValidatorChain{validators: []preparedValidator{validator(), validator()}},
}
got, err := collectPreparedCheckpointFingerprints(prepared)
if err != nil {
t.Fatalf("collect repeated validators: %v", err)
}
if len(got) != 2 || got[0].Name == got[1].Name {
t.Fatalf("fingerprints = %#v, want independently scoped repeated validators", got)
}
}
type checkpointFingerprintTestProvider []CheckpointFingerprint
func (p checkpointFingerprintTestProvider) CheckpointFingerprints() []CheckpointFingerprint {
return append([]CheckpointFingerprint(nil), p...)
}
func (checkpointFingerprintTestProvider) Key() string { return "fingerprint-test" }
func (checkpointFingerprintTestProvider) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (checkpointFingerprintTestProvider) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) {
return typedTestDocument(), nil
}
func (checkpointFingerprintTestProvider) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
return contracts.ChunkPlanResult{}, nil
}
func (checkpointFingerprintTestProvider) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
func (checkpointFingerprintTestProvider) Name() string { return "fingerprint-test" }
func (checkpointFingerprintTestProvider) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (checkpointFingerprintTestProvider) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
type checkpointFingerprintOutput struct {
contracts.OutputEncoder
fingerprints []CheckpointFingerprint
}
func (o checkpointFingerprintOutput) CheckpointFingerprints() []CheckpointFingerprint {
return append([]CheckpointFingerprint(nil), o.fingerprints...)
}
func fingerprintTestValidatorChain(key string, provider checkpointFingerprintTestProvider) preparedValidatorChain {
return preparedValidatorChain{validators: []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding(key), Target: ValidatorTargetTyped},
typed: provider,
}}}
}
func TestPrepareDeliversTargetReferencesAsIndependentBuildInputs(t *testing.T) {
var built []string
var observations []constructionBuildObservation
@@ -183,9 +344,26 @@ func TestPrepareFailuresOccurBeforeInputParse(t *testing.T) {
}
}
func TestPrepareRejectsInvalidComponentCheckpointFingerprint(t *testing.T) {
failure := &constructionFailure{outputFingerprints: []CheckpointFingerprint{{Name: "identity"}}}
registries, input := constructionRegistries(t, nil, failure)
resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog())
if err != nil {
t.Fatal(err)
}
_, err = Prepare(resolved, registries, ModuleDependencies{})
if err == nil || !strings.Contains(err.Error(), "checkpoint fingerprint") || !strings.Contains(err.Error(), "output:output") {
t.Fatalf("Prepare() error = %v, want scoped checkpoint fingerprint error", err)
}
if len(input.requests) != 0 {
t.Fatalf("input Parse calls = %d, want zero", len(input.requests))
}
}
type constructionFailure struct {
requireExtractorLLM bool
output error
outputFingerprints []CheckpointFingerprint
}
func constructionProfile() PipelineProfile {
@@ -298,7 +476,11 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
if failure.output != nil {
return nil, failure.output
}
return &typedTestOutput{key: "output"}, nil
output := contracts.OutputEncoder(&typedTestOutput{key: "output"})
if failure.outputFingerprints != nil {
output = checkpointFingerprintOutput{OutputEncoder: output, fingerprints: failure.outputFingerprints}
}
return output, nil
}); err != nil {
t.Fatal(err)
}

View File

@@ -17,13 +17,14 @@ type PreparedPipeline struct {
ArtifactLanes []PreparedArtifactLane
Output ModuleBinding
resolved ResolvedPipeline
dependencies ModuleDependencies
input contracts.InputAdapter
chunker contracts.Chunker
chunkValidators preparedValidatorChain
lanes []preparedLaneExecutor
output contracts.OutputEncoder
resolved ResolvedPipeline
dependencies ModuleDependencies
input contracts.InputAdapter
chunker contracts.Chunker
chunkValidators preparedValidatorChain
lanes []preparedLaneExecutor
output contracts.OutputEncoder
checkpointFingerprints []CheckpointFingerprint
}
type PreparedArtifactLane struct {
@@ -114,6 +115,10 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
return nil, constructionError(stable.ID, "", StageOutput, stable.Output.Module, "", err)
}
prepared.output = output
prepared.checkpointFingerprints, err = collectPreparedCheckpointFingerprints(prepared)
if err != nil {
return nil, err
}
return prepared, nil
}

View File

@@ -0,0 +1,86 @@
package pipeline
import (
"fmt"
"sort"
"strings"
)
type checkpointFingerprintComponent struct {
scope string
module any
}
// CheckpointFingerprints returns a defensive copy of the stable semantic
// identities contributed by the pipeline's prepared modules and validators.
func (p *PreparedPipeline) CheckpointFingerprints() []CheckpointFingerprint {
if p == nil {
return nil
}
return append([]CheckpointFingerprint(nil), p.checkpointFingerprints...)
}
func collectPreparedCheckpointFingerprints(prepared *PreparedPipeline) ([]CheckpointFingerprint, error) {
components := []checkpointFingerprintComponent{
{scope: "input:" + prepared.resolved.Input.Module, module: prepared.input},
{scope: "chunk:" + prepared.resolved.Chunk.Module, module: prepared.chunker},
}
components = appendValidatorFingerprintComponents(components, "chunk:"+prepared.resolved.Chunk.Module, prepared.chunkValidators)
for _, lane := range prepared.lanes {
laneScope := func(stage ModuleStage, moduleKey string) string {
return string(stage) + ":" + lane.resolved.ID + ":" + moduleKey
}
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageExtract, lane.resolved.Extract.Module), module: lane.typed.extractor})
components = appendValidatorFingerprintComponents(components, laneScope(StageExtract, lane.resolved.Extract.Module), lane.extractValidators)
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageMerge, lane.resolved.Merge.Module), module: lane.typed.merger})
components = appendValidatorFingerprintComponents(components, laneScope(StageMerge, lane.resolved.Merge.Module), lane.mergeValidators)
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageNormalize, lane.resolved.Normalize.Module), module: lane.typed.normalizer})
components = appendValidatorFingerprintComponents(components, laneScope(StageNormalize, lane.resolved.Normalize.Module), lane.normalizeValidators)
}
components = append(components, checkpointFingerprintComponent{scope: "output:" + prepared.resolved.Output.Module, module: prepared.output})
var values []CheckpointFingerprint
seen := make(map[string]struct{})
for _, item := range components {
provider, ok := item.module.(CheckpointFingerprintProvider)
if !ok {
continue
}
for _, fingerprint := range provider.CheckpointFingerprints() {
name := strings.TrimSpace(fingerprint.Name)
value := strings.TrimSpace(fingerprint.Value)
if name == "" || value == "" {
return nil, fmt.Errorf("pipeline %q checkpoint fingerprint for %s must have a non-empty name and value", prepared.resolved.ID, item.scope)
}
scopedName := item.scope + ":" + name
if _, exists := seen[scopedName]; exists {
return nil, fmt.Errorf("pipeline %q checkpoint fingerprint %q is duplicated", prepared.resolved.ID, scopedName)
}
seen[scopedName] = struct{}{}
values = append(values, CheckpointFingerprint{Name: scopedName, Value: value})
}
}
sort.Slice(values, func(i, j int) bool { return values[i].Name < values[j].Name })
return values, nil
}
func appendValidatorFingerprintComponents(components []checkpointFingerprintComponent, ownerScope string, chain preparedValidatorChain) []checkpointFingerprintComponent {
for index, validator := range chain.validators {
scope := fmt.Sprintf("%s:validator:%d:%s", ownerScope, index, validator.resolved.Binding.Module)
components = append(components, checkpointFingerprintComponent{scope: scope, module: preparedValidatorImplementation(validator)})
}
return components
}
func preparedValidatorImplementation(validator preparedValidator) any {
switch validator.resolved.Target {
case ValidatorTargetTyped:
return validator.typed
case ValidatorTargetChunk:
return validator.chunk
case ValidatorTargetSerialized:
return validator.serialized
default:
return nil
}
}

View File

@@ -43,6 +43,7 @@ func referenceSlots() []contracts.ReferenceSlot {
}
var _ contracts.Extractor[dnd.SpellList] = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
@@ -109,6 +110,13 @@ func (e *Extractor) ManifestMetadata() map[string]any {
return metadata
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{{Name: "effective_catalog", Value: e.effectiveCatalog.Digest()}}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("extractor must not be nil")

View File

@@ -117,6 +117,10 @@ func TestExtractPromptUsesCanonicalOverlayNamesWithoutAliasesOrMetadata(t *testi
if got, ok := metadata["catalog_overlay_ids"].([]string); !ok || !reflect.DeepEqual(got, []string{"campaign.example"}) {
t.Fatalf("catalog overlay metadata = %#v", metadata["catalog_overlay_ids"])
}
fingerprints := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).CheckpointFingerprints()
if len(fingerprints) != 1 || fingerprints[0].Name != "effective_catalog" || fingerprints[0].Value != metadata["catalog_digest"] {
t.Fatalf("checkpoint fingerprints = %#v, want manifest catalog digest %#v", fingerprints, metadata["catalog_digest"])
}
encoded, err := json.Marshal(metadata)
if err != nil {
t.Fatal(err)

View File

@@ -13,9 +13,11 @@ import (
)
const (
Key = "extract/dnd/spells/catalog"
ReasonCode = "unknown_spell"
maxIssues = 20
Key = "extract/dnd/spells/catalog"
ReasonCode = "unknown_spell"
maxIssues = 20
maxDisplayedNameRunes = 128
maxMessageBytes = 4096
)
type Options struct{}
@@ -25,6 +27,7 @@ type Validator struct {
}
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
func New(_ Options, references ...contracts.ReferenceSet) (*Validator, error) {
if len(references) > 1 {
@@ -47,6 +50,13 @@ func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if v == nil {
return nil
}
return []pipeline.CheckpointFingerprint{{Name: "effective_catalog", Value: v.catalog.Digest()}}
}
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
if err := spellshape.Validate(req.Value); err != nil {
return contracts.ValidationResult{Approved: true}, nil
@@ -74,19 +84,37 @@ type unknownSpell struct {
}
func rejection(unknown []unknownSpell) contracts.ValidationResult {
displayed := unknown
if len(displayed) > maxIssues {
displayed = displayed[:maxIssues]
limit := len(unknown)
if limit > maxIssues {
limit = maxIssues
}
issues := make([]string, len(displayed))
for index, item := range displayed {
issues[index] = fmt.Sprintf("spell_casts[%d].spell %q", item.index, item.name)
issues := make([]string, 0, limit)
for _, item := range unknown[:limit] {
issue := fmt.Sprintf("spell_casts[%d].spell %q", item.index, truncateDisplayedName(item.name))
candidate := rejectionMessage(append(issues, issue), len(unknown)-len(issues)-1)
if len(candidate) > maxMessageBytes {
break
}
issues = append(issues, issue)
}
message := fmt.Sprintf("unknown spell names: %s", strings.Join(issues, ", "))
if omitted := len(unknown) - len(displayed); omitted > 0 {
message := rejectionMessage(issues, len(unknown)-len(issues))
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
}
func truncateDisplayedName(name string) string {
runes := []rune(name)
if len(runes) <= maxDisplayedNameRunes {
return name
}
return string(runes[:maxDisplayedNameRunes-1]) + "…"
}
func rejectionMessage(issues []string, omitted int) string {
message := "unknown spell names: " + strings.Join(issues, ", ")
if omitted > 0 {
message += fmt.Sprintf("; %d additional issue(s) omitted", omitted)
}
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
return message
}
func Spec() pipeline.ValidatorSpec {

View File

@@ -6,6 +6,7 @@ import (
"reflect"
"strings"
"testing"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -29,6 +30,17 @@ func TestValidatorApprovesCanonicalNormalizedAndOverlayAliasNames(t *testing.T)
}
}
func TestValidatorCheckpointFingerprintUsesEffectiveCatalogDigest(t *testing.T) {
validator, err := New(Options{}, overlayReferences())
if err != nil {
t.Fatal(err)
}
fingerprints := validator.CheckpointFingerprints()
if len(fingerprints) != 1 || fingerprints[0].Name != "effective_catalog" || fingerprints[0].Value != validator.catalog.Digest() {
t.Fatalf("checkpoint fingerprints = %#v, want effective catalog digest %q", fingerprints, validator.catalog.Digest())
}
}
func TestValidatorRejectsMultipleUnknownCastsInStableOrder(t *testing.T) {
value := spellList("Unknown First", "Cure Wounds", "Unknown Second")
validator, err := New(Options{})
@@ -71,6 +83,50 @@ func TestValidatorBoundsUnknownCastMessage(t *testing.T) {
}
}
func TestValidatorBoundsUnknownSpellNamesAndTotalMessage(t *testing.T) {
longName := strings.Repeat("火", maxDisplayedNameRunes+100) + "\n\t"
value := dnd.SpellList{SpellCasts: make([]dnd.SpellCast, maxIssues)}
for index := range value.SpellCasts {
value.SpellCasts[index] = validCast(fmt.Sprintf("%s-%d", longName, index))
}
validator, err := New(Options{})
if err != nil {
t.Fatal(err)
}
result, err := validator.Validate(context.Background(), validationRequest(value))
if err != nil {
t.Fatal(err)
}
if result.Approved || len([]byte(result.Message)) > maxMessageBytes || !strings.Contains(result.Message, "…") {
t.Fatalf("message length/content = %d/%q, want bounded message with truncation", len([]byte(result.Message)), result.Message)
}
if !strings.Contains(result.Message, "additional issue(s) omitted") {
t.Fatalf("message = %q, want byte-budget omitted count", result.Message)
}
displayed := strings.Count(result.Message, "spell_casts[")
wantOmitted := fmt.Sprintf("%d additional issue(s) omitted", len(value.SpellCasts)-displayed)
if !strings.Contains(result.Message, wantOmitted) {
t.Fatalf("message = %q, want omitted count %q", result.Message, wantOmitted)
}
if !utf8.ValidString(result.Message) {
t.Fatal("bounded message is not valid UTF-8")
}
}
func TestValidatorQuotesControlCharactersInUnknownSpellName(t *testing.T) {
validator, err := New(Options{})
if err != nil {
t.Fatal(err)
}
result, err := validator.Validate(context.Background(), validationRequest(spellList("Unknown\nSpell\tName")))
if err != nil {
t.Fatal(err)
}
if strings.Contains(result.Message, "Unknown\nSpell") || !strings.Contains(result.Message, `Unknown\nSpell\tName`) {
t.Fatalf("message = %q, want safely quoted control characters", result.Message)
}
}
func TestValidatorDefersInvalidShape(t *testing.T) {
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{{Spell: "Unknown Spell"}}}
validator, err := New(Options{})