package cli import ( "crypto/sha256" "encoding/hex" "encoding/json" "os" "path/filepath" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog" ) func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) { components := productionTestComponents(t) configPath := repositoryPath("examples", "dnd-spells-production.config.yml") effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve production configuration: %v", err) } overlayPath := filepath.Join(t.TempDir(), "catalog.json") resolved := effective.ResolvedPipeline bindings := resolved.ArtifactLanes[0].ExtractReferences.Bindings catalogBindingIndex := -1 for index, binding := range bindings { if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot { catalogBindingIndex = index break } } if catalogBindingIndex < 0 { t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings) } resolved.ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil { t.Fatal(err) } materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) if err != nil { t.Fatalf("materialize first catalog: %v", err) } identityA := catalogCheckpointIdentity(t, materializedA) metadataA := catalogExtractorMetadata(t, materializedA) referenceA := catalogReference(t, materializedA) if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil { t.Fatal(err) } materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) if err != nil { t.Fatalf("materialize reordered catalog: %v", err) } identityB := catalogCheckpointIdentity(t, materializedB) metadataB := catalogExtractorMetadata(t, materializedB) referenceB := catalogReference(t, materializedB) if identityA.Digest == identityB.Digest { t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest) } if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI { t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB) } digestA, ok := metadataA["catalog_digest"].(string) if !ok { t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA) } digestB, ok := metadataB["catalog_digest"].(string) if !ok || digestA != digestB { t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB) } if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) { t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want) } } func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) { base := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml")) changed := strings.Replace(base, "./dnd-spells-catalog.json", "./alternate-spell-catalog.json", 1) if changed == base { t.Fatal("production configuration did not contain the maintained catalog binding") } root := t.TempDir() firstPath := filepath.Join(root, "first.yml") secondPath := filepath.Join(root, "second.yml") if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil { t.Fatal(err) } components := productionTestComponents(t) first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve first configuration: %v", err) } second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve changed configuration: %v", err) } if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest { t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest) } } func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) { components := productionTestComponents(t) configPath := repositoryPath("examples", "dnd-spells-production.config.yml") effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatal(err) } materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) if err != nil { t.Fatal(err) } prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) if err != nil { t.Fatal(err) } fingerprints := prepared.CheckpointFingerprints() if len(fingerprints) != 2 || fingerprints[0].Value != fingerprints[1].Value { t.Fatalf("prepared fingerprints = %#v, want matching extractor and validator catalog identities", fingerprints) } identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity { identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{ Pipeline: materialized, InputKey: materialized.Input.Module, RawInputDigest: "sha256:unchanged-input", References: pipeline.ReferenceProvenance(materialized), ProvenanceFingerprints: checkpointIdentityFingerprints(values), }) if identityErr != nil { t.Fatal(identityErr) } return identity } first := identityFor(fingerprints) changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...) changed[0].Value = "sha256:changed-effective-catalog" second := identityFor(changed) if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) { t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second) } } func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) { components := productionTestComponents(t) configPath := repositoryPath("examples", "dnd-spells-production.config.yml") effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatal(err) } materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) if err != nil { t.Fatal(err) } prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) if err != nil { t.Fatal(err) } fingerprints := prepared.CheckpointFingerprints() settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()} recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false) if err != nil { t.Fatal(err) } doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"} doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} doc.Digest, err = source.DigestDocument(&doc) if err != nil { t.Fatal(err) } if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil { t.Fatal(err) } _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true) if err != nil { t.Fatal(err) } if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused { t.Fatalf("same fingerprint decision = %#v, want reuse", decision) } changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...) changed[0].Value = "sha256:changed-effective-catalog" _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true) if err != nil { t.Fatal(err) } if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused { t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision) } } func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")} options := productionRunOptions(t, fake) var stdout, stderr strings.Builder code := RunWithOptions([]string{ "run", "dnd-session", "--config", repositoryPath("examples", "dnd-spells-production.config.yml"), "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, }, &stdout, &stderr, options) if code != 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } runRoot := filepath.Join(outputRoot, productionRunID) manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json")) if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 { t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest) } lane := manifest.ArtifactLanes[0] extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) if !ok { t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata) } if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") { t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata) } if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) { t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got) } var catalogProvenance *artifacts.ReferenceProvenance for index := range manifest.References { reference := &manifest.References[index] if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot { catalogProvenance = reference break } } if catalogProvenance == nil { t.Fatalf("manifest references = %#v, want spell catalog provenance", manifest.References) } overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json") if catalogProvenance.Stage != "extract" || catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") { t.Fatalf("catalog provenance = %#v, want extract origin, media, size, and raw digest", catalogProvenance) } manifestBytes, err := json.Marshal(manifest) if err != nil { t.Fatal(err) } for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} { if strings.Contains(string(manifestBytes), leaked) { t.Fatalf("manifest leaked overlay content %q", leaked) } } requests := fake.requestsFor(spells.PromptID) if len(requests) != 1 { t.Fatalf("spell requests = %d, want one", len(requests)) } catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot] if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") { t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput) } artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json")) if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" { t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact) } rejected := readProductionJSON[struct { Rejected []json.RawMessage `json:"rejected"` }](t, filepath.Join(runRoot, "rejected.json")) if len(rejected.Rejected) != 0 { t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected) } } func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity { t.Helper() identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{ Pipeline: resolved, InputKey: resolved.Input.Module, RawInputDigest: "sha256:catalog-test-input", References: pipeline.ReferenceProvenance(resolved), }) if err != nil { t.Fatalf("create checkpoint identity: %v", err) } return identity } func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any { t.Helper() lane := resolved.ArtifactLanes[0] extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet) if err != nil { t.Fatalf("construct extractor: %v", err) } return extractor.ManifestMetadata() } func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance { t.Helper() for _, reference := range pipeline.ReferenceProvenance(resolved) { if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" { return reference } } t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved)) return artifacts.ReferenceProvenance{} } func stringValue(value any) string { result, _ := value.(string) return result } func stringValues(value any) []string { raw, err := json.Marshal(value) if err != nil { return nil } var values []string if err := json.Unmarshal(raw, &values); err != nil { return nil } return values } func digestBytes(value []byte) string { sum := sha256.Sum256(value) return "sha256:" + hex.EncodeToString(sum[:]) } const reorderedOverlayA = `{ "schema_version": "notarius.dnd.spell-catalog-overlay.v1", "catalogs": [ {"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]}, {"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]} ] }` const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`