diff --git a/internal/cli/example_contract_test.go b/internal/cli/example_contract_test.go index 9fcab48..73fcc1d 100644 --- a/internal/cli/example_contract_test.go +++ b/internal/cli/example_contract_test.go @@ -83,6 +83,13 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) { if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != "noop" { t.Fatalf("manifest lane = %#v, want production spells composition", lane) } + if len(manifest.References) != 0 { + t.Fatalf("base-only manifest references = %#v, want no overlay provenance", manifest.References) + } + extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) + if !ok || len(stringValues(extractorMetadata["catalog_overlay_ids"])) != 0 { + t.Fatalf("base-only extractor metadata = %#v, want no overlay IDs", lane.Metadata) + } artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, entry.File)) if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" { diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index e8c9859..2b36ded 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -455,8 +455,9 @@ func readProductionJSON[T any](t *testing.T, path string) T { } type productionFakeLLMClient struct { - mu sync.Mutex - requests []contracts.StructuredCompletionRequest + mu sync.Mutex + requests []contracts.StructuredCompletionRequest + spellResponse string } func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { @@ -468,7 +469,11 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r case scenes.PromptID: content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Opening scene","primary_mode":"Narrative","main_participants":["Aria"],"summary":"The session opens.","boundary_note":"The opening covers the available transcript.","boundary_confidence":"High"}],"boundary_caveats":["The opening boundary is inferred from the short transcript."]}`) case spells.PromptID: - content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`) + if client.spellResponse != "" { + content = []byte(client.spellResponse) + } else { + content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`) + } default: return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID) } diff --git a/internal/cli/spell_catalog_identity_contract_test.go b/internal/cli/spell_catalog_identity_contract_test.go new file mode 100644 index 0000000..606c316 --- /dev/null +++ b/internal/cli/spell_catalog_identity_contract_test.go @@ -0,0 +1,255 @@ +package cli + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" + spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog" +) + +func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) { + components := productionTestComponents(t) + configPath := repositoryPath("examples", "dnd-spells-production.config.yml") + effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + if err != nil { + t.Fatalf("resolve production configuration: %v", err) + } + overlayPath := filepath.Join(t.TempDir(), "catalog.json") + resolved := effective.ResolvedPipeline + bindings := resolved.ArtifactLanes[0].ExtractReferences.Bindings + catalogBindingIndex := -1 + for index, binding := range bindings { + if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot { + catalogBindingIndex = index + break + } + } + if catalogBindingIndex < 0 { + t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings) + } + resolved.ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath + + if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil { + t.Fatal(err) + } + materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) + if err != nil { + t.Fatalf("materialize first catalog: %v", err) + } + identityA := catalogCheckpointIdentity(t, materializedA) + metadataA := catalogExtractorMetadata(t, materializedA) + referenceA := catalogReference(t, materializedA) + + if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil { + t.Fatal(err) + } + materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)}) + if err != nil { + t.Fatalf("materialize reordered catalog: %v", err) + } + identityB := catalogCheckpointIdentity(t, materializedB) + metadataB := catalogExtractorMetadata(t, materializedB) + referenceB := catalogReference(t, materializedB) + + if identityA.Digest == identityB.Digest { + t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest) + } + if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI { + t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB) + } + digestA, ok := metadataA["catalog_digest"].(string) + if !ok { + t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA) + } + digestB, ok := metadataB["catalog_digest"].(string) + if !ok || digestA != digestB { + t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB) + } + if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) { + t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want) + } +} + +func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) { + base := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml")) + changed := strings.Replace(base, "./dnd-spells-catalog.json", "./alternate-spell-catalog.json", 1) + if changed == base { + t.Fatal("production configuration did not contain the maintained catalog binding") + } + root := t.TempDir() + firstPath := filepath.Join(root, "first.yml") + secondPath := filepath.Join(root, "second.yml") + if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil { + t.Fatal(err) + } + components := productionTestComponents(t) + first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + if err != nil { + t.Fatalf("resolve first configuration: %v", err) + } + second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) + if err != nil { + t.Fatalf("resolve changed configuration: %v", err) + } + if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest { + t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest) + } +} + +func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) { + outputRoot := filepath.Join(t.TempDir(), "output") + fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")} + options := productionRunOptions(t, fake) + var stdout, stderr strings.Builder + code := RunWithOptions([]string{ + "run", "dnd-session", + "--config", repositoryPath("examples", "dnd-spells-production.config.yml"), + "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), + "--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, + }, &stdout, &stderr, options) + if code != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + + runRoot := filepath.Join(outputRoot, productionRunID) + manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json")) + if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 { + t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest) + } + lane := manifest.ArtifactLanes[0] + extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) + if !ok { + t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata) + } + if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") { + t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata) + } + if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) { + t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got) + } + + var catalogProvenance *artifacts.ReferenceProvenance + for index := range manifest.References { + reference := &manifest.References[index] + if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot { + catalogProvenance = reference + break + } + } + if catalogProvenance == nil { + t.Fatalf("manifest references = %#v, want spell catalog provenance", manifest.References) + } + overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json") + if catalogProvenance.Stage != "extract" || catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") { + t.Fatalf("catalog provenance = %#v, want extract origin, media, size, and raw digest", catalogProvenance) + } + manifestBytes, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} { + if strings.Contains(string(manifestBytes), leaked) { + t.Fatalf("manifest leaked overlay content %q", leaked) + } + } + + requests := fake.requestsFor(spells.PromptID) + if len(requests) != 1 { + t.Fatalf("spell requests = %d, want one", len(requests)) + } + catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot] + if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") { + t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput) + } + artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json")) + if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" { + t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact) + } + rejected := readProductionJSON[struct { + Rejected []json.RawMessage `json:"rejected"` + }](t, filepath.Join(runRoot, "rejected.json")) + if len(rejected.Rejected) != 0 { + t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected) + } +} + +func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity { + t.Helper() + identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{ + Pipeline: resolved, + InputKey: resolved.Input.Module, + RawInputDigest: "sha256:catalog-test-input", + References: pipeline.ReferenceProvenance(resolved), + }) + if err != nil { + t.Fatalf("create checkpoint identity: %v", err) + } + return identity +} + +func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any { + t.Helper() + lane := resolved.ArtifactLanes[0] + extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet) + if err != nil { + t.Fatalf("construct extractor: %v", err) + } + return extractor.ManifestMetadata() +} + +func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance { + t.Helper() + for _, reference := range pipeline.ReferenceProvenance(resolved) { + if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" { + return reference + } + } + t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved)) + return artifacts.ReferenceProvenance{} +} + +func stringValue(value any) string { + result, _ := value.(string) + return result +} + +func stringValues(value any) []string { + raw, err := json.Marshal(value) + if err != nil { + return nil + } + var values []string + if err := json.Unmarshal(raw, &values); err != nil { + return nil + } + return values +} + +func digestBytes(value []byte) string { + sum := sha256.Sum256(value) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +const reorderedOverlayA = `{ + "schema_version": "notarius.dnd.spell-catalog-overlay.v1", + "catalogs": [ + {"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]}, + {"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]} + ] +}` + +const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`