diff --git a/docs/internal/modules.md b/docs/internal/modules.md index eee303e..429f9d3 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -158,7 +158,12 @@ reference into an immutable effective catalog and adds a generated canonical-name-only JSON input to every structured completion request. Overlay failures therefore stop construction before source parsing or an LLM call; campaign references remain separate disambiguation inputs and never become -source evidence. The separate `internal/modules/dnd/codec/spells` package +source evidence. Both the extractor and deterministic catalog validator expose +the effective base-plus-overlay semantic digest as scoped prepared-component +checkpoint identity. Raw overlay provenance independently covers file-byte +changes, while the semantic digest also invalidates reuse when the embedded +catalog or catalog composition changes. The separate +`internal/modules/dnd/codec/spells` package owns the durable schema and stable JSON representation for artifact kind `dnd/spell-list`. The runner keeps the result typed through validators and later stages, using the codec only for checkpoint, debug, and output boundaries. diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 7a6b36a..d7544ec 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -126,8 +126,16 @@ then output. It stops at the first error with pipeline, stage, lane, module, and validator context as applicable. It never invokes an operation method. `PreparedPipeline` keeps private constructed executors and exposes cloned -resolved input, chunk, lane, and output identities. `pipeline.RunInput` carries -that prepared pipeline, raw source input, run identity and timing, optional +resolved input, chunk, lane, and output identities. Prepared components may +implement `pipeline.CheckpointFingerprintProvider` to contribute explicit +semantic identities to checkpoint reuse. Preparation trims and validates each +non-secret name and value, prefixes it with the component's stage, lane, +module, and validator scope, rejects duplicates, and retains the resulting +sorted collection behind a defensive-copy accessor. Fingerprints must be +stable and must not contain source content, credentials, local paths, +timestamps, or other invocation-specific values. + +`pipeline.RunInput` carries that prepared pipeline, raw source input, run identity and timing, optional session and profile metadata, a chunk-plan store and mode, and checkpoint/debug collaborators. The runner parses source bytes through the already constructed input adapter. Later stage @@ -267,6 +275,14 @@ bytes, and decode failures become explicit reuse misses and execute the step normally. Dependency fingerprints and debug content digests use the same stable codec bytes that cross those boundaries. +The CLI includes prepared-component fingerprints in the run-wide checkpoint +identity alongside resolved configuration, raw input, reference provenance, +runtime overrides, and LLM-profile fingerprints. Module metadata is not used +implicitly for cache identity: components opt in only with stable semantic +values that can change accepted output. Adding or changing a component +fingerprint intentionally produces a cold cache miss. Existing checkpoint +schemas and paths remain unchanged. + Debug instrumentation wraps run, stage, attempt, validator, and structured LLM boundaries. Every executed chunk, extract, merge, and normalize attempt writes one terminal envelope for acceptance, validator rejection, module or validator diff --git a/docs/internal/state.md b/docs/internal/state.md index 7b3da1d..86403db 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -25,6 +25,9 @@ not `bypass`. `internal/framework/checkpoint` owns checkpoint identity, manifests, payload codecs, loader, and recorder. The CLI constructs a recorder whenever checkpoint recording is enabled and constructs a loader only for a `--resume` invocation. +Identity incorporates explicit stable semantic fingerprints collected from +prepared modules and validators in addition to configuration, input, +references, runtime overrides, and LLM profiles. The serialized `workspace_schema_version` identifiers are frozen wire-compatibility fields; they do not describe a current public state surface. diff --git a/docs/operations.md b/docs/operations.md index 4a88815..b7604ec 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -89,6 +89,11 @@ Checkpoints use the selected root and the existing identity hierarchy: //-///... ``` +The final identity digest includes stable semantic fingerprints explicitly +contributed by prepared modules and validators. Adding or changing one of +these fingerprints intentionally causes a cold cache miss; old checkpoint +directories are left in place and are never migrated or deleted automatically. + An empty configured directory selects `/notarius/checkpoints`. The root is exact when configured. Created directories and files use `0700` and `0600` permissions on supported diff --git a/docs/roadmap/dnd-spell-extraction.md b/docs/roadmap/dnd-spell-extraction.md index 8df0656..4c9c0bf 100644 --- a/docs/roadmap/dnd-spell-extraction.md +++ b/docs/roadmap/dnd-spell-extraction.md @@ -6,7 +6,10 @@ The accepted baseline implementation is complete. It provides effective SRD-plus-overlay catalog composition, canonical-name prompt grounding, deterministic catalog validation, the maintained `retries: 2` policy, and provenance and checkpoint identity coverage through the assembled offline -workflow. +workflow. Checkpoint identity includes independently scoped extractor and +validator fingerprints of the effective catalog, closing the earlier gap in +which raw overlay changes invalidated reuse but embedded catalog changes did +not. External quality evaluation is pending. This repository contains the maintained example and offline fake-LLM coverage, but no approved diff --git a/internal/cli/run.go b/internal/cli/run.go index ec664e1..76025f5 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -352,7 +352,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } - checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume) + checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume) if err != nil { return failPipelineCommand(stderr, commandState, terminalWriter, err) } @@ -434,6 +434,7 @@ func checkpointHandlersForRun( settings config.CheckpointCacheConfig, opts Options, resolved pipeline.ResolvedPipeline, + componentFingerprints []pipeline.CheckpointFingerprint, rawInput []byte, only []string, llmProfiles []artifacts.LLMProfileManifest, @@ -454,7 +455,7 @@ func checkpointHandlersForRun( SelectedLanes: only, RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID), References: pipeline.ReferenceProvenance(resolved), - ProvenanceFingerprints: llmProfileFingerprints(llmProfiles), + ProvenanceFingerprints: append(llmProfileFingerprints(llmProfiles), checkpointIdentityFingerprints(componentFingerprints)...), }) if err != nil { return nil, nil, fmt.Errorf("create checkpoint identity: %w", err) @@ -480,6 +481,17 @@ func checkpointHandlersForRun( return recorder, loader, nil } +func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint { + if len(values) == 0 { + return nil + } + out := make([]checkpoint.Fingerprint, len(values)) + for index, value := range values { + out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value} + } + return out +} + func rawInputDigest(data []byte) string { sum := sha256.Sum256(data) return "sha256:" + hex.EncodeToString(sum[:]) diff --git a/internal/cli/spell_catalog_identity_contract_test.go b/internal/cli/spell_catalog_identity_contract_test.go index 606c316..c1f9202 100644 --- a/internal/cli/spell_catalog_identity_contract_test.go +++ b/internal/cli/spell_catalog_identity_contract_test.go @@ -11,6 +11,8 @@ import ( "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" @@ -110,6 +112,97 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing } } +func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) { + components := productionTestComponents(t) + configPath := repositoryPath("examples", "dnd-spells-production.config.yml") + effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + if err != nil { + t.Fatal(err) + } + materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) + if err != nil { + t.Fatal(err) + } + prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) + if err != nil { + t.Fatal(err) + } + fingerprints := prepared.CheckpointFingerprints() + if len(fingerprints) != 2 || fingerprints[0].Value != fingerprints[1].Value { + t.Fatalf("prepared fingerprints = %#v, want matching extractor and validator catalog identities", fingerprints) + } + + identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity { + identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{ + Pipeline: materialized, + InputKey: materialized.Input.Module, + RawInputDigest: "sha256:unchanged-input", + References: pipeline.ReferenceProvenance(materialized), + ProvenanceFingerprints: checkpointIdentityFingerprints(values), + }) + if identityErr != nil { + t.Fatal(identityErr) + } + return identity + } + first := identityFor(fingerprints) + changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...) + changed[0].Value = "sha256:changed-effective-catalog" + second := identityFor(changed) + if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) { + t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second) + } +} + +func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) { + components := productionTestComponents(t) + configPath := repositoryPath("examples", "dnd-spells-production.config.yml") + effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + if err != nil { + t.Fatal(err) + } + materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) + if err != nil { + t.Fatal(err) + } + prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) + if err != nil { + t.Fatal(err) + } + fingerprints := prepared.CheckpointFingerprints() + settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()} + recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false) + if err != nil { + t.Fatal(err) + } + doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"} + doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} + doc.Digest, err = source.DigestDocument(&doc) + if err != nil { + t.Fatal(err) + } + if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil { + t.Fatal(err) + } + + _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true) + if err != nil { + t.Fatal(err) + } + if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused { + t.Fatalf("same fingerprint decision = %#v, want reuse", decision) + } + changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...) + changed[0].Value = "sha256:changed-effective-catalog" + _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true) + if err != nil { + t.Fatal(err) + } + if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused { + t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision) + } +} + func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")} diff --git a/internal/framework/pipeline/checkpoint.go b/internal/framework/pipeline/checkpoint.go index 429e817..7803848 100644 --- a/internal/framework/pipeline/checkpoint.go +++ b/internal/framework/pipeline/checkpoint.go @@ -16,6 +16,13 @@ type CheckpointFingerprint struct { Value string `json:"value"` } +// CheckpointFingerprintProvider supplies stable, non-secret semantic identity +// for a prepared module or validator. Values must not contain source content, +// credentials, local paths, timestamps, or other invocation-specific data. +type CheckpointFingerprintProvider interface { + CheckpointFingerprints() []CheckpointFingerprint +} + type CheckpointRecorder interface { SourceRunning(moduleKey string) error SourceSucceeded(moduleKey string, doc *source.SourceDocument) error diff --git a/internal/framework/pipeline/preparation_test.go b/internal/framework/pipeline/preparation_test.go index 4db807d..6585167 100644 --- a/internal/framework/pipeline/preparation_test.go +++ b/internal/framework/pipeline/preparation_test.go @@ -72,6 +72,167 @@ func TestPrepareConstructsEverythingInStableOrder(t *testing.T) { } } +func TestPreparedCheckpointFingerprintsCollectEveryComponentInStableScopeOrder(t *testing.T) { + provider := func(value string) checkpointFingerprintTestProvider { + return checkpointFingerprintTestProvider{{Name: "identity", Value: value}} + } + prepared := &PreparedPipeline{ + resolved: ResolvedPipeline{ + ID: "fingerprints", + Input: Binding("input"), + Chunk: Binding("chunk"), + Output: Binding("output"), + }, + input: provider("input"), + chunker: provider("chunk"), + chunkValidators: preparedValidatorChain{validators: []preparedValidator{{ + resolved: ResolvedValidator{Binding: Binding("chunk-validator"), Target: ValidatorTargetChunk}, + chunk: provider("chunk-validator"), + }}}, + output: provider("output"), + } + lane := preparedLaneExecutor{ + resolved: ResolvedArtifactLane{ + ID: "spells", + Extract: Binding("extract"), + Merge: Binding("merge"), + Normalize: Binding("normalize"), + }, + typed: &preparedTypedLane{ + extractor: provider("extract"), + merger: provider("merge"), + normalizer: provider("normalize"), + }, + extractValidators: fingerprintTestValidatorChain("extract-validator", provider("extract-validator")), + mergeValidators: fingerprintTestValidatorChain("merge-validator", provider("merge-validator")), + normalizeValidators: fingerprintTestValidatorChain("normalize-validator", provider("normalize-validator")), + } + prepared.lanes = []preparedLaneExecutor{lane} + + first, err := collectPreparedCheckpointFingerprints(prepared) + if err != nil { + t.Fatalf("collectPreparedCheckpointFingerprints() error = %v, want nil", err) + } + second, err := collectPreparedCheckpointFingerprints(prepared) + if err != nil { + t.Fatalf("second collection error = %v, want nil", err) + } + if !reflect.DeepEqual(first, second) { + t.Fatalf("fingerprints are not deterministic: %#v / %#v", first, second) + } + if len(first) != 10 { + t.Fatalf("fingerprints = %#v, want all 10 prepared components", first) + } + for index := 1; index < len(first); index++ { + if first[index-1].Name >= first[index].Name { + t.Fatalf("fingerprints are not sorted: %#v", first) + } + } + prepared.checkpointFingerprints = first + copy := prepared.CheckpointFingerprints() + copy[0].Value = "mutated" + if reflect.DeepEqual(copy, prepared.CheckpointFingerprints()) { + t.Fatal("CheckpointFingerprints() exposed mutable backing storage") + } +} + +func TestPreparedCheckpointFingerprintsValidateProviderValues(t *testing.T) { + tests := []struct { + name string + fingerprints []CheckpointFingerprint + want string + }{ + {name: "empty name", fingerprints: []CheckpointFingerprint{{Value: "value"}}, want: "non-empty name and value"}, + {name: "empty value", fingerprints: []CheckpointFingerprint{{Name: "name"}}, want: "non-empty name and value"}, + {name: "duplicate after trimming", fingerprints: []CheckpointFingerprint{{Name: "same", Value: "one"}, {Name: " same ", Value: "two"}}, want: "duplicated"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + prepared := &PreparedPipeline{ + resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")}, + input: checkpointFingerprintTestProvider(test.fingerprints), + } + _, err := collectPreparedCheckpointFingerprints(prepared) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want substring %q", err, test.want) + } + }) + } +} + +func TestPreparedCheckpointFingerprintsRemainEmptyWithoutProviders(t *testing.T) { + registries, _ := constructionRegistries(t, nil, nil) + resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog()) + if err != nil { + t.Fatal(err) + } + prepared, err := Prepare(resolved, registries, ModuleDependencies{}) + if err != nil { + t.Fatal(err) + } + if got := prepared.CheckpointFingerprints(); len(got) != 0 { + t.Fatalf("CheckpointFingerprints() = %#v, want empty", got) + } +} + +func TestPreparedCheckpointFingerprintsScopeRepeatedValidatorsByPosition(t *testing.T) { + provider := checkpointFingerprintTestProvider{{Name: "identity", Value: "same"}} + validator := func() preparedValidator { + return preparedValidator{resolved: ResolvedValidator{Binding: Binding("configured"), Target: ValidatorTargetChunk}, chunk: provider} + } + prepared := &PreparedPipeline{ + resolved: ResolvedPipeline{ID: "fingerprints", Input: Binding("input"), Chunk: Binding("chunk"), Output: Binding("output")}, + chunkValidators: preparedValidatorChain{validators: []preparedValidator{validator(), validator()}}, + } + got, err := collectPreparedCheckpointFingerprints(prepared) + if err != nil { + t.Fatalf("collect repeated validators: %v", err) + } + if len(got) != 2 || got[0].Name == got[1].Name { + t.Fatalf("fingerprints = %#v, want independently scoped repeated validators", got) + } +} + +type checkpointFingerprintTestProvider []CheckpointFingerprint + +func (p checkpointFingerprintTestProvider) CheckpointFingerprints() []CheckpointFingerprint { + return append([]CheckpointFingerprint(nil), p...) +} +func (checkpointFingerprintTestProvider) Key() string { return "fingerprint-test" } +func (checkpointFingerprintTestProvider) ReferenceSlots() []contracts.ReferenceSlot { return nil } +func (checkpointFingerprintTestProvider) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) { + return typedTestDocument(), nil +} +func (checkpointFingerprintTestProvider) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { + return contracts.ChunkPlanResult{}, nil +} +func (checkpointFingerprintTestProvider) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) { + return contracts.OutputResult{}, nil +} +func (checkpointFingerprintTestProvider) Name() string { return "fingerprint-test" } +func (checkpointFingerprintTestProvider) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} +func (checkpointFingerprintTestProvider) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { + return contracts.ValidationResult{Approved: true}, nil +} + +type checkpointFingerprintOutput struct { + contracts.OutputEncoder + fingerprints []CheckpointFingerprint +} + +func (o checkpointFingerprintOutput) CheckpointFingerprints() []CheckpointFingerprint { + return append([]CheckpointFingerprint(nil), o.fingerprints...) +} + +func fingerprintTestValidatorChain(key string, provider checkpointFingerprintTestProvider) preparedValidatorChain { + return preparedValidatorChain{validators: []preparedValidator{{ + resolved: ResolvedValidator{Binding: Binding(key), Target: ValidatorTargetTyped}, + typed: provider, + }}} +} + func TestPrepareDeliversTargetReferencesAsIndependentBuildInputs(t *testing.T) { var built []string var observations []constructionBuildObservation @@ -183,9 +344,26 @@ func TestPrepareFailuresOccurBeforeInputParse(t *testing.T) { } } +func TestPrepareRejectsInvalidComponentCheckpointFingerprint(t *testing.T) { + failure := &constructionFailure{outputFingerprints: []CheckpointFingerprint{{Name: "identity"}}} + registries, input := constructionRegistries(t, nil, failure) + resolved, err := ResolvePipeline(constructionProfile(), ResolveOptions{}, registries.catalog()) + if err != nil { + t.Fatal(err) + } + _, err = Prepare(resolved, registries, ModuleDependencies{}) + if err == nil || !strings.Contains(err.Error(), "checkpoint fingerprint") || !strings.Contains(err.Error(), "output:output") { + t.Fatalf("Prepare() error = %v, want scoped checkpoint fingerprint error", err) + } + if len(input.requests) != 0 { + t.Fatalf("input Parse calls = %d, want zero", len(input.requests)) + } +} + type constructionFailure struct { requireExtractorLLM bool output error + outputFingerprints []CheckpointFingerprint } func constructionProfile() PipelineProfile { @@ -298,7 +476,11 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con if failure.output != nil { return nil, failure.output } - return &typedTestOutput{key: "output"}, nil + output := contracts.OutputEncoder(&typedTestOutput{key: "output"}) + if failure.outputFingerprints != nil { + output = checkpointFingerprintOutput{OutputEncoder: output, fingerprints: failure.outputFingerprints} + } + return output, nil }); err != nil { t.Fatal(err) } diff --git a/internal/framework/pipeline/prepare.go b/internal/framework/pipeline/prepare.go index aaa9f2f..76ea45e 100644 --- a/internal/framework/pipeline/prepare.go +++ b/internal/framework/pipeline/prepare.go @@ -17,13 +17,14 @@ type PreparedPipeline struct { ArtifactLanes []PreparedArtifactLane Output ModuleBinding - resolved ResolvedPipeline - dependencies ModuleDependencies - input contracts.InputAdapter - chunker contracts.Chunker - chunkValidators preparedValidatorChain - lanes []preparedLaneExecutor - output contracts.OutputEncoder + resolved ResolvedPipeline + dependencies ModuleDependencies + input contracts.InputAdapter + chunker contracts.Chunker + chunkValidators preparedValidatorChain + lanes []preparedLaneExecutor + output contracts.OutputEncoder + checkpointFingerprints []CheckpointFingerprint } type PreparedArtifactLane struct { @@ -114,6 +115,10 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend return nil, constructionError(stable.ID, "", StageOutput, stable.Output.Module, "", err) } prepared.output = output + prepared.checkpointFingerprints, err = collectPreparedCheckpointFingerprints(prepared) + if err != nil { + return nil, err + } return prepared, nil } diff --git a/internal/framework/pipeline/prepared_fingerprints.go b/internal/framework/pipeline/prepared_fingerprints.go new file mode 100644 index 0000000..238abea --- /dev/null +++ b/internal/framework/pipeline/prepared_fingerprints.go @@ -0,0 +1,86 @@ +package pipeline + +import ( + "fmt" + "sort" + "strings" +) + +type checkpointFingerprintComponent struct { + scope string + module any +} + +// CheckpointFingerprints returns a defensive copy of the stable semantic +// identities contributed by the pipeline's prepared modules and validators. +func (p *PreparedPipeline) CheckpointFingerprints() []CheckpointFingerprint { + if p == nil { + return nil + } + return append([]CheckpointFingerprint(nil), p.checkpointFingerprints...) +} + +func collectPreparedCheckpointFingerprints(prepared *PreparedPipeline) ([]CheckpointFingerprint, error) { + components := []checkpointFingerprintComponent{ + {scope: "input:" + prepared.resolved.Input.Module, module: prepared.input}, + {scope: "chunk:" + prepared.resolved.Chunk.Module, module: prepared.chunker}, + } + components = appendValidatorFingerprintComponents(components, "chunk:"+prepared.resolved.Chunk.Module, prepared.chunkValidators) + for _, lane := range prepared.lanes { + laneScope := func(stage ModuleStage, moduleKey string) string { + return string(stage) + ":" + lane.resolved.ID + ":" + moduleKey + } + components = append(components, checkpointFingerprintComponent{scope: laneScope(StageExtract, lane.resolved.Extract.Module), module: lane.typed.extractor}) + components = appendValidatorFingerprintComponents(components, laneScope(StageExtract, lane.resolved.Extract.Module), lane.extractValidators) + components = append(components, checkpointFingerprintComponent{scope: laneScope(StageMerge, lane.resolved.Merge.Module), module: lane.typed.merger}) + components = appendValidatorFingerprintComponents(components, laneScope(StageMerge, lane.resolved.Merge.Module), lane.mergeValidators) + components = append(components, checkpointFingerprintComponent{scope: laneScope(StageNormalize, lane.resolved.Normalize.Module), module: lane.typed.normalizer}) + components = appendValidatorFingerprintComponents(components, laneScope(StageNormalize, lane.resolved.Normalize.Module), lane.normalizeValidators) + } + components = append(components, checkpointFingerprintComponent{scope: "output:" + prepared.resolved.Output.Module, module: prepared.output}) + + var values []CheckpointFingerprint + seen := make(map[string]struct{}) + for _, item := range components { + provider, ok := item.module.(CheckpointFingerprintProvider) + if !ok { + continue + } + for _, fingerprint := range provider.CheckpointFingerprints() { + name := strings.TrimSpace(fingerprint.Name) + value := strings.TrimSpace(fingerprint.Value) + if name == "" || value == "" { + return nil, fmt.Errorf("pipeline %q checkpoint fingerprint for %s must have a non-empty name and value", prepared.resolved.ID, item.scope) + } + scopedName := item.scope + ":" + name + if _, exists := seen[scopedName]; exists { + return nil, fmt.Errorf("pipeline %q checkpoint fingerprint %q is duplicated", prepared.resolved.ID, scopedName) + } + seen[scopedName] = struct{}{} + values = append(values, CheckpointFingerprint{Name: scopedName, Value: value}) + } + } + sort.Slice(values, func(i, j int) bool { return values[i].Name < values[j].Name }) + return values, nil +} + +func appendValidatorFingerprintComponents(components []checkpointFingerprintComponent, ownerScope string, chain preparedValidatorChain) []checkpointFingerprintComponent { + for index, validator := range chain.validators { + scope := fmt.Sprintf("%s:validator:%d:%s", ownerScope, index, validator.resolved.Binding.Module) + components = append(components, checkpointFingerprintComponent{scope: scope, module: preparedValidatorImplementation(validator)}) + } + return components +} + +func preparedValidatorImplementation(validator preparedValidator) any { + switch validator.resolved.Target { + case ValidatorTargetTyped: + return validator.typed + case ValidatorTargetChunk: + return validator.chunk + case ValidatorTargetSerialized: + return validator.serialized + default: + return nil + } +} diff --git a/internal/modules/dnd/extract/spells/extractor.go b/internal/modules/dnd/extract/spells/extractor.go index 6b10ea8..a22d4b4 100644 --- a/internal/modules/dnd/extract/spells/extractor.go +++ b/internal/modules/dnd/extract/spells/extractor.go @@ -43,6 +43,7 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.SpellList] = (*Extractor)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} @@ -109,6 +110,13 @@ func (e *Extractor) ManifestMetadata() map[string]any { return metadata } +func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + if e == nil { + return nil + } + return []pipeline.CheckpointFingerprint{{Name: "effective_catalog", Value: e.effectiveCatalog.Digest()}} +} + func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) { if e == nil { return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("extractor must not be nil") diff --git a/internal/modules/dnd/extract/spells/extractor_test.go b/internal/modules/dnd/extract/spells/extractor_test.go index 7d4791c..2681721 100644 --- a/internal/modules/dnd/extract/spells/extractor_test.go +++ b/internal/modules/dnd/extract/spells/extractor_test.go @@ -117,6 +117,10 @@ func TestExtractPromptUsesCanonicalOverlayNamesWithoutAliasesOrMetadata(t *testi if got, ok := metadata["catalog_overlay_ids"].([]string); !ok || !reflect.DeepEqual(got, []string{"campaign.example"}) { t.Fatalf("catalog overlay metadata = %#v", metadata["catalog_overlay_ids"]) } + fingerprints := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).CheckpointFingerprints() + if len(fingerprints) != 1 || fingerprints[0].Name != "effective_catalog" || fingerprints[0].Value != metadata["catalog_digest"] { + t.Fatalf("checkpoint fingerprints = %#v, want manifest catalog digest %#v", fingerprints, metadata["catalog_digest"]) + } encoded, err := json.Marshal(metadata) if err != nil { t.Fatal(err) diff --git a/internal/modules/dnd/validate/spells/catalog/validator.go b/internal/modules/dnd/validate/spells/catalog/validator.go index 09364b4..c90f3c3 100644 --- a/internal/modules/dnd/validate/spells/catalog/validator.go +++ b/internal/modules/dnd/validate/spells/catalog/validator.go @@ -13,9 +13,11 @@ import ( ) const ( - Key = "extract/dnd/spells/catalog" - ReasonCode = "unknown_spell" - maxIssues = 20 + Key = "extract/dnd/spells/catalog" + ReasonCode = "unknown_spell" + maxIssues = 20 + maxDisplayedNameRunes = 128 + maxMessageBytes = 4096 ) type Options struct{} @@ -25,6 +27,7 @@ type Validator struct { } var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil) func New(_ Options, references ...contracts.ReferenceSet) (*Validator, error) { if len(references) > 1 { @@ -47,6 +50,13 @@ func (v *Validator) ExecutionClass() contracts.ExecutionClass { return contracts.ExecutionClassDeterministic } +func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + if v == nil { + return nil + } + return []pipeline.CheckpointFingerprint{{Name: "effective_catalog", Value: v.catalog.Digest()}} +} + func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) { if err := spellshape.Validate(req.Value); err != nil { return contracts.ValidationResult{Approved: true}, nil @@ -74,19 +84,37 @@ type unknownSpell struct { } func rejection(unknown []unknownSpell) contracts.ValidationResult { - displayed := unknown - if len(displayed) > maxIssues { - displayed = displayed[:maxIssues] + limit := len(unknown) + if limit > maxIssues { + limit = maxIssues } - issues := make([]string, len(displayed)) - for index, item := range displayed { - issues[index] = fmt.Sprintf("spell_casts[%d].spell %q", item.index, item.name) + issues := make([]string, 0, limit) + for _, item := range unknown[:limit] { + issue := fmt.Sprintf("spell_casts[%d].spell %q", item.index, truncateDisplayedName(item.name)) + candidate := rejectionMessage(append(issues, issue), len(unknown)-len(issues)-1) + if len(candidate) > maxMessageBytes { + break + } + issues = append(issues, issue) } - message := fmt.Sprintf("unknown spell names: %s", strings.Join(issues, ", ")) - if omitted := len(unknown) - len(displayed); omitted > 0 { + message := rejectionMessage(issues, len(unknown)-len(issues)) + return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message} +} + +func truncateDisplayedName(name string) string { + runes := []rune(name) + if len(runes) <= maxDisplayedNameRunes { + return name + } + return string(runes[:maxDisplayedNameRunes-1]) + "…" +} + +func rejectionMessage(issues []string, omitted int) string { + message := "unknown spell names: " + strings.Join(issues, ", ") + if omitted > 0 { message += fmt.Sprintf("; %d additional issue(s) omitted", omitted) } - return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message} + return message } func Spec() pipeline.ValidatorSpec { diff --git a/internal/modules/dnd/validate/spells/catalog/validator_test.go b/internal/modules/dnd/validate/spells/catalog/validator_test.go index ae233cc..08f223b 100644 --- a/internal/modules/dnd/validate/spells/catalog/validator_test.go +++ b/internal/modules/dnd/validate/spells/catalog/validator_test.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" "testing" + "unicode/utf8" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" @@ -29,6 +30,17 @@ func TestValidatorApprovesCanonicalNormalizedAndOverlayAliasNames(t *testing.T) } } +func TestValidatorCheckpointFingerprintUsesEffectiveCatalogDigest(t *testing.T) { + validator, err := New(Options{}, overlayReferences()) + if err != nil { + t.Fatal(err) + } + fingerprints := validator.CheckpointFingerprints() + if len(fingerprints) != 1 || fingerprints[0].Name != "effective_catalog" || fingerprints[0].Value != validator.catalog.Digest() { + t.Fatalf("checkpoint fingerprints = %#v, want effective catalog digest %q", fingerprints, validator.catalog.Digest()) + } +} + func TestValidatorRejectsMultipleUnknownCastsInStableOrder(t *testing.T) { value := spellList("Unknown First", "Cure Wounds", "Unknown Second") validator, err := New(Options{}) @@ -71,6 +83,50 @@ func TestValidatorBoundsUnknownCastMessage(t *testing.T) { } } +func TestValidatorBoundsUnknownSpellNamesAndTotalMessage(t *testing.T) { + longName := strings.Repeat("火", maxDisplayedNameRunes+100) + "\n\t" + value := dnd.SpellList{SpellCasts: make([]dnd.SpellCast, maxIssues)} + for index := range value.SpellCasts { + value.SpellCasts[index] = validCast(fmt.Sprintf("%s-%d", longName, index)) + } + validator, err := New(Options{}) + if err != nil { + t.Fatal(err) + } + result, err := validator.Validate(context.Background(), validationRequest(value)) + if err != nil { + t.Fatal(err) + } + if result.Approved || len([]byte(result.Message)) > maxMessageBytes || !strings.Contains(result.Message, "…") { + t.Fatalf("message length/content = %d/%q, want bounded message with truncation", len([]byte(result.Message)), result.Message) + } + if !strings.Contains(result.Message, "additional issue(s) omitted") { + t.Fatalf("message = %q, want byte-budget omitted count", result.Message) + } + displayed := strings.Count(result.Message, "spell_casts[") + wantOmitted := fmt.Sprintf("%d additional issue(s) omitted", len(value.SpellCasts)-displayed) + if !strings.Contains(result.Message, wantOmitted) { + t.Fatalf("message = %q, want omitted count %q", result.Message, wantOmitted) + } + if !utf8.ValidString(result.Message) { + t.Fatal("bounded message is not valid UTF-8") + } +} + +func TestValidatorQuotesControlCharactersInUnknownSpellName(t *testing.T) { + validator, err := New(Options{}) + if err != nil { + t.Fatal(err) + } + result, err := validator.Validate(context.Background(), validationRequest(spellList("Unknown\nSpell\tName"))) + if err != nil { + t.Fatal(err) + } + if strings.Contains(result.Message, "Unknown\nSpell") || !strings.Contains(result.Message, `Unknown\nSpell\tName`) { + t.Fatalf("message = %q, want safely quoted control characters", result.Message) + } +} + func TestValidatorDefersInvalidShape(t *testing.T) { value := dnd.SpellList{SpellCasts: []dnd.SpellCast{{Spell: "Unknown Spell"}}} validator, err := New(Options{})