package cli import ( "crypto/sha256" "encoding/hex" "encoding/json" "os" "path/filepath" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog" ) func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) { components := productionTestComponents(t) configPath := writeProductionSpellCatalogContractConfig(t) effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve production configuration: %v", err) } overlayPath := filepath.Join(t.TempDir(), "catalog.json") resolved := effective.ResolvedPipeline bindings := resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings catalogBindingIndex := -1 for index, binding := range bindings { if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot { catalogBindingIndex = index break } } if catalogBindingIndex < 0 { t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings) } resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath normalizeBindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings normalizeCatalogBindingIndex := -1 for index, binding := range normalizeBindings { if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot { normalizeCatalogBindingIndex = index break } } if normalizeCatalogBindingIndex < 0 { t.Fatalf("normalize spell catalog bindings = %#v, want catalog binding", normalizeBindings) } resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings[normalizeCatalogBindingIndex].Source = overlayPath if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil { t.Fatal(err) } materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) if err != nil { t.Fatalf("materialize first catalog: %v", err) } identityA := catalogCheckpointIdentity(t, materializedA) metadataA := catalogExtractorMetadata(t, materializedA) normalizerMetadataA := catalogNormalizerMetadata(t, materializedA) referenceA := catalogReference(t, materializedA) if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil { t.Fatal(err) } materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) if err != nil { t.Fatalf("materialize reordered catalog: %v", err) } identityB := catalogCheckpointIdentity(t, materializedB) metadataB := catalogExtractorMetadata(t, materializedB) normalizerMetadataB := catalogNormalizerMetadata(t, materializedB) referenceB := catalogReference(t, materializedB) if identityA.Digest == identityB.Digest { t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest) } if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI { t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB) } digestA, ok := metadataA["catalog_digest"].(string) if !ok { t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA) } digestB, ok := metadataB["catalog_digest"].(string) if !ok || digestA != digestB { t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB) } if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) { t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want) } normalizerDigestA, ok := normalizerMetadataA["catalog_digest"].(string) normalizerDigestB, okB := normalizerMetadataB["catalog_digest"].(string) if !ok || !okB || normalizerDigestA != digestA || normalizerDigestB != digestB { t.Fatalf("normalizer catalog digests = %#v and %#v, want extractor semantic digests %q and %q", normalizerMetadataA["catalog_digest"], normalizerMetadataB["catalog_digest"], digestA, digestB) } if got, want := normalizerMetadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(normalizerMetadataB["catalog_overlay_ids"], want) { t.Fatalf("normalizer overlay IDs = %#v and %#v, want %#v", got, normalizerMetadataB["catalog_overlay_ids"], want) } } func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) { base := productionSpellCatalogContractConfig(t) changed := strings.Replace(base, repositoryPath("examples", "dnd-spell-catalog.json"), filepath.Join(t.TempDir(), "alternate-spell-catalog.json"), 1) if changed == base { t.Fatal("production configuration did not contain the maintained catalog binding") } root := t.TempDir() firstPath := filepath.Join(root, "first.yml") secondPath := filepath.Join(root, "second.yml") if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil { t.Fatal(err) } components := productionTestComponents(t) first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve first configuration: %v", err) } second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve changed configuration: %v", err) } if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest { t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest) } } func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) { components := productionTestComponents(t) configPath := writeProductionSpellCatalogContractConfig(t) effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatal(err) } materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) if err != nil { t.Fatal(err) } prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) if err != nil { t.Fatal(err) } fingerprints := prepared.CheckpointFingerprints() wantNames := map[string]struct{}{ "extract:spells:" + spells.Key + ":effective_catalog": {}, "extract:spells:" + spells.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {}, "normalize:spells:" + spellnormalize.Key + ":effective_catalog": {}, "normalize:spells:" + spellnormalize.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {}, } seen := make(map[string]string, len(fingerprints)) for _, fingerprint := range fingerprints { if _, ok := wantNames[fingerprint.Name]; ok { seen[fingerprint.Name] = fingerprint.Value } } if len(seen) != len(wantNames) { t.Fatalf("prepared fingerprints = %#v, want scoped extractor and normalize catalog identities", fingerprints) } var catalogDigest string for name, value := range seen { if catalogDigest == "" { catalogDigest = value } else if value != catalogDigest { t.Fatalf("prepared fingerprint %q = %q, want shared semantic catalog digest %q", name, value, catalogDigest) } } identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity { identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{ Pipeline: materialized, InputKey: materialized.Input.Module, RawInputDigest: "sha256:unchanged-input", References: pipeline.ReferenceProvenance(materialized), ProvenanceFingerprints: checkpointIdentityFingerprints(values), }) if identityErr != nil { t.Fatal(identityErr) } return identity } first := identityFor(fingerprints) changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog") assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName()) second := identityFor(changed) if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) { t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second) } } func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) { components := productionTestComponents(t) configPath := writeProductionSpellCatalogContractConfig(t) effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatal(err) } materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) if err != nil { t.Fatal(err) } prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) if err != nil { t.Fatal(err) } fingerprints := prepared.CheckpointFingerprints() settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()} recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false) if err != nil { t.Fatal(err) } doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"} doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} doc.Digest, err = source.DigestDocument(&doc) if err != nil { t.Fatal(err) } if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil { t.Fatal(err) } normalizeDependencies := []pipeline.CheckpointFingerprint{{Name: "artifact[0]", Value: "sha256:merged-artifact"}} normalizeSchema := contracts.ArtifactSchema{ID: "notarius.dnd.spells", Name: "notarius_dnd_spells", Version: "v1"} normalizeArtifact := pipeline.CheckpointArtifact{ LaneID: "spells", ModuleKey: spellnormalize.Key, SourceID: doc.ID, SchemaDigest: contracts.DigestArtifactSchema(normalizeSchema), Artifact: contracts.SerializedArtifact{ Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), }, } if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil { t.Fatal(err) } _, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true) if err != nil { t.Fatal(err) } if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused { t.Fatalf("same fingerprint decision = %#v, want reuse", decision) } if restored, decision := sameLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); !decision.Reused || string(restored.Output.Artifact.Content) != `{"spell_casts":[]}` { t.Fatalf("same normalize checkpoint = %#v, decision=%#v, want reuse", restored, decision) } changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog") assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName()) _, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true) if err != nil { t.Fatal(err) } if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused { t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision) } if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused { t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision) } changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3") assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName()) _, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, []byte("same input"), nil, nil, "", "", true) if err != nil { t.Fatal(err) } if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused { t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision) } } func normalizeSpellCatalogFingerprintName() string { return "normalize:spells:" + spellnormalize.Key + ":effective_catalog" } func extractSpellMappingFingerprintName() string { return "extract:spells:" + spells.Key + ":mapping_policy" } func replaceCheckpointFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, value string) []pipeline.CheckpointFingerprint { t.Helper() changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...) matches := 0 for index := range changed { if changed[index].Name == name { changed[index].Value = value matches++ } } if matches != 1 { t.Fatalf("checkpoint fingerprints = %#v, want exactly one fingerprint named %q", fingerprints, name) } return changed } func assertOnlyCheckpointFingerprintChanged(t *testing.T, before, after []pipeline.CheckpointFingerprint, changedName string) { t.Helper() if len(before) != len(after) { t.Fatalf("fingerprint lengths = %d and %d, want equal", len(before), len(after)) } changes := 0 for index := range before { if before[index].Name != after[index].Name { t.Fatalf("fingerprint[%d] name changed from %q to %q", index, before[index].Name, after[index].Name) } if before[index].Value == after[index].Value { continue } changes++ if before[index].Name != changedName { t.Fatalf("fingerprint %q changed unexpectedly", before[index].Name) } } if changes != 1 { t.Fatalf("fingerprints changed %d values, want exactly %q", changes, changedName) } } func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")} options := productionRunOptions(t, fake) var stdout, stderr strings.Builder code := RunWithOptions([]string{ "run", "dnd-session", "--config", writeProductionSpellCatalogContractConfig(t), "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, }, &stdout, &stderr, options) if code != 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } runRoot := filepath.Join(outputRoot, productionRunID) manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json")) if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 { t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest) } lane := manifest.ArtifactLanes[0] extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) if !ok { t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata) } if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") { t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata) } if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) { t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got) } normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any) if !ok { t.Fatalf("lane metadata = %#v, want normalizer metadata", lane.Metadata) } if normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), []string{"notarius.example-campaign"}) { t.Fatalf("normalizer catalog metadata = %#v, want base ID, semantic digest, and overlay IDs", normalizerMetadata) } if normalizerMetadata["catalog_digest"] != extractorMetadata["catalog_digest"] || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), stringValues(extractorMetadata["catalog_overlay_ids"])) { t.Fatalf("extractor metadata = %#v, normalizer metadata = %#v, want shared catalog identity", extractorMetadata, normalizerMetadata) } var catalogProvenances []artifacts.ReferenceProvenance for index := range manifest.References { reference := &manifest.References[index] if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot { catalogProvenances = append(catalogProvenances, *reference) } } if len(catalogProvenances) != 2 { t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References) } overlayBytes := readRepositoryFile(t, "examples", "dnd-spell-catalog.json") for _, catalogProvenance := range catalogProvenances { if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" { t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance) } if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spell-catalog.json") { t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance) } } manifestBytes, err := json.Marshal(manifest) if err != nil { t.Fatal(err) } for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} { if strings.Contains(string(manifestBytes), leaked) { t.Fatalf("manifest leaked overlay content %q", leaked) } } requests := fake.requestsFor(spells.PromptID) if len(requests) != 1 { t.Fatalf("spell requests = %d, want one", len(requests)) } catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot] if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") { t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput) } artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json")) if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" { t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact) } rejected := readProductionJSON[struct { Rejected []json.RawMessage `json:"rejected"` }](t, filepath.Join(runRoot, "rejected.json")) if len(rejected.Rejected) != 0 { t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected) } } func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity { t.Helper() identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{ Pipeline: resolved, InputKey: resolved.Input.Module, RawInputDigest: "sha256:catalog-test-input", References: pipeline.ReferenceProvenance(resolved), }) if err != nil { t.Fatalf("create checkpoint identity: %v", err) } return identity } func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any { t.Helper() lane := resolved.Steps[0].ArtifactLanes[0] extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet) if err != nil { t.Fatalf("construct extractor: %v", err) } return extractor.ManifestMetadata() } func catalogNormalizerMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any { t.Helper() lane := resolved.Steps[0].ArtifactLanes[0] normalizer, err := spellnormalize.New(spellnormalize.Options{}, lane.NormalizeReferences.ReferenceSet) if err != nil { t.Fatalf("construct normalizer: %v", err) } return normalizer.ManifestMetadata() } func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance { t.Helper() for _, reference := range pipeline.ReferenceProvenance(resolved) { if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" { return reference } } t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved)) return artifacts.ReferenceProvenance{} } func stringValue(value any) string { result, _ := value.(string) return result } func stringValues(value any) []string { raw, err := json.Marshal(value) if err != nil { return nil } var values []string if err := json.Unmarshal(raw, &values); err != nil { return nil } return values } func digestBytes(value []byte) string { sum := sha256.Sum256(value) return "sha256:" + hex.EncodeToString(sum[:]) } const reorderedOverlayA = `{ "schema_version": "notarius.dnd.spell-catalog-overlay.v1", "catalogs": [ {"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]}, {"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]} ] }` const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`