Verify spell catalog provenance and checkpoint identity

internal/cli/example_contract_test.go

@@ -83,6 +83,13 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
 	if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != "noop" {
 		t.Fatalf("manifest lane = %#v, want production spells composition", lane)
 	}
+	if len(manifest.References) != 0 {
+		t.Fatalf("base-only manifest references = %#v, want no overlay provenance", manifest.References)
+	}
+	extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
+	if !ok || len(stringValues(extractorMetadata["catalog_overlay_ids"])) != 0 {
+		t.Fatalf("base-only extractor metadata = %#v, want no overlay IDs", lane.Metadata)
+	}
 
 	artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, entry.File))
 	if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" {

internal/cli/production_contract_test.go

@@ -455,8 +455,9 @@ func readProductionJSON[T any](t *testing.T, path string) T {
 }
 
 type productionFakeLLMClient struct {
-	mu       sync.Mutex
+	mu            sync.Mutex
-	requests []contracts.StructuredCompletionRequest
+	requests      []contracts.StructuredCompletionRequest
+	spellResponse string
 }
 
 func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -468,7 +469,11 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
 	case scenes.PromptID:
 		content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Opening scene","primary_mode":"Narrative","main_participants":["Aria"],"summary":"The session opens.","boundary_note":"The opening covers the available transcript.","boundary_confidence":"High"}],"boundary_caveats":["The opening boundary is inferred from the short transcript."]}`)
 	case spells.PromptID:
-		content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`)
+		if client.spellResponse != "" {
+			content = []byte(client.spellResponse)
+		} else {
+			content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`)
+		}
 	default:
 		return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
 	}

internal/cli/spell_catalog_identity_contract_test.go

@@ -0,0 +1,255 @@
+package cli
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
+	spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
+)
+
+func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
+	components := productionTestComponents(t)
+	configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
+	effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
+	if err != nil {
+		t.Fatalf("resolve production configuration: %v", err)
+	}
+	overlayPath := filepath.Join(t.TempDir(), "catalog.json")
+	resolved := effective.ResolvedPipeline
+	bindings := resolved.ArtifactLanes[0].ExtractReferences.Bindings
+	catalogBindingIndex := -1
+	for index, binding := range bindings {
+		if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
+			catalogBindingIndex = index
+			break
+		}
+	}
+	if catalogBindingIndex < 0 {
+		t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
+	}
+	resolved.ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
+
+	if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
+	if err != nil {
+		t.Fatalf("materialize first catalog: %v", err)
+	}
+	identityA := catalogCheckpointIdentity(t, materializedA)
+	metadataA := catalogExtractorMetadata(t, materializedA)
+	referenceA := catalogReference(t, materializedA)
+
+	if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
+	if err != nil {
+		t.Fatalf("materialize reordered catalog: %v", err)
+	}
+	identityB := catalogCheckpointIdentity(t, materializedB)
+	metadataB := catalogExtractorMetadata(t, materializedB)
+	referenceB := catalogReference(t, materializedB)
+
+	if identityA.Digest == identityB.Digest {
+		t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest)
+	}
+	if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI {
+		t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB)
+	}
+	digestA, ok := metadataA["catalog_digest"].(string)
+	if !ok {
+		t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA)
+	}
+	digestB, ok := metadataB["catalog_digest"].(string)
+	if !ok || digestA != digestB {
+		t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB)
+	}
+	if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
+		t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
+	}
+}
+
+func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
+	base := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml"))
+	changed := strings.Replace(base, "./dnd-spells-catalog.json", "./alternate-spell-catalog.json", 1)
+	if changed == base {
+		t.Fatal("production configuration did not contain the maintained catalog binding")
+	}
+	root := t.TempDir()
+	firstPath := filepath.Join(root, "first.yml")
+	secondPath := filepath.Join(root, "second.yml")
+	if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	components := productionTestComponents(t)
+	first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
+	if err != nil {
+		t.Fatalf("resolve first configuration: %v", err)
+	}
+	second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
+	if err != nil {
+		t.Fatalf("resolve changed configuration: %v", err)
+	}
+	if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest {
+		t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest)
+	}
+}
+
+func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
+	outputRoot := filepath.Join(t.TempDir(), "output")
+	fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
+	options := productionRunOptions(t, fake)
+	var stdout, stderr strings.Builder
+	code := RunWithOptions([]string{
+		"run", "dnd-session",
+		"--config", repositoryPath("examples", "dnd-spells-production.config.yml"),
+		"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
+		"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
+	}, &stdout, &stderr, options)
+	if code != 0 {
+		t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
+	}
+
+	runRoot := filepath.Join(outputRoot, productionRunID)
+	manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
+	if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 {
+		t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest)
+	}
+	lane := manifest.ArtifactLanes[0]
+	extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
+	if !ok {
+		t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
+	}
+	if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") {
+		t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata)
+	}
+	if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
+		t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
+	}
+
+	var catalogProvenance *artifacts.ReferenceProvenance
+	for index := range manifest.References {
+		reference := &manifest.References[index]
+		if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
+			catalogProvenance = reference
+			break
+		}
+	}
+	if catalogProvenance == nil {
+		t.Fatalf("manifest references = %#v, want spell catalog provenance", manifest.References)
+	}
+	overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json")
+	if catalogProvenance.Stage != "extract" || catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") {
+		t.Fatalf("catalog provenance = %#v, want extract origin, media, size, and raw digest", catalogProvenance)
+	}
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} {
+		if strings.Contains(string(manifestBytes), leaked) {
+			t.Fatalf("manifest leaked overlay content %q", leaked)
+		}
+	}
+
+	requests := fake.requestsFor(spells.PromptID)
+	if len(requests) != 1 {
+		t.Fatalf("spell requests = %d, want one", len(requests))
+	}
+	catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
+	if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") {
+		t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput)
+	}
+	artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
+	if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
+		t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact)
+	}
+	rejected := readProductionJSON[struct {
+		Rejected []json.RawMessage `json:"rejected"`
+	}](t, filepath.Join(runRoot, "rejected.json"))
+	if len(rejected.Rejected) != 0 {
+		t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected)
+	}
+}
+
+func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity {
+	t.Helper()
+	identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
+		Pipeline:       resolved,
+		InputKey:       resolved.Input.Module,
+		RawInputDigest: "sha256:catalog-test-input",
+		References:     pipeline.ReferenceProvenance(resolved),
+	})
+	if err != nil {
+		t.Fatalf("create checkpoint identity: %v", err)
+	}
+	return identity
+}
+
+func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
+	t.Helper()
+	lane := resolved.ArtifactLanes[0]
+	extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet)
+	if err != nil {
+		t.Fatalf("construct extractor: %v", err)
+	}
+	return extractor.ManifestMetadata()
+}
+
+func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
+	t.Helper()
+	for _, reference := range pipeline.ReferenceProvenance(resolved) {
+		if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" {
+			return reference
+		}
+	}
+	t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved))
+	return artifacts.ReferenceProvenance{}
+}
+
+func stringValue(value any) string {
+	result, _ := value.(string)
+	return result
+}
+
+func stringValues(value any) []string {
+	raw, err := json.Marshal(value)
+	if err != nil {
+		return nil
+	}
+	var values []string
+	if err := json.Unmarshal(raw, &values); err != nil {
+		return nil
+	}
+	return values
+}
+
+func digestBytes(value []byte) string {
+	sum := sha256.Sum256(value)
+	return "sha256:" + hex.EncodeToString(sum[:])
+}
+
+const reorderedOverlayA = `{
+  "schema_version": "notarius.dnd.spell-catalog-overlay.v1",
+  "catalogs": [
+    {"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]},
+    {"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]}
+  ]
+}`
+
+const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`