Harden item occurrence grounding

docs/integrations/dnd-item-occurrence-artifacts.md

@@ -27,11 +27,13 @@ an earlier normalized `dnd/item-registry` artifact. The registry is immutable
 for an operation and contributes only its ordered `{id,name}` projection after
 the shared evidence message. It is never occurrence evidence.
 
-Each occurrence must use one exact registry ID/name pair. Extraction omits an
+Each occurrence must use one exact registry ID/name pair. An extraction response
-unknown ID or mismatched name rather than creating an item. Normalization
+with an unknown ID or mismatched name is rejected as invalid model output; the
-canonicalizes a recognized name by ID, preserves unknown values for the
+configured pipeline may retry it and never accepts a partial artifact.
-registry validator, and the registry validator rejects unknown or mismatched
+Normalization and validation remain defense in depth for artifacts entering
-pairs.
+through other boundaries: normalization canonicalizes a recognized name by ID,
+preserves unknown values for the registry validator, and the registry validator
+rejects unknown or mismatched pairs.
 
 ## Wire shape
 

docs/internal/dnd.md

@@ -160,6 +160,11 @@ These projections are guidance only and never event evidence.
 The following differences are intentional and should remain explicit when a
 shared helper changes.
 
+Shared D&D text comparison is identified by `dnd.text_comparison.v1`. Any
+semantic change requires an explicit policy-version review for every affected
+identity, mapping, normalization, and validator policy; helper source is not a
+checkpoint fingerprint.
+
 | Lane | Intentional behavior |
 | --- | --- |
 | Spells | May use a spell-catalog overlay and optional NPC grounding; the catalog validator supplies domain-specific semantic checks. |

internal/modules/dnd/extract/itemoccurrences/canonicalize.go

@@ -1,6 +1,7 @@
 package itemoccurrences
 
 import (
+	"fmt"
 	"sort"
 
 	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -15,12 +16,15 @@ type orderedItemOccurrenceResponse struct {
 	hasEvidence bool
 }
 
-func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string) {
+func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string, registry *itemregistry.Registry) error {
 	if response == nil {
-		return
+		return nil
 	}
 	ordered := make([]orderedItemOccurrenceResponse, len(response.Occurrences))
 	for index := range response.Occurrences {
+		if err := validateRegistryPair(index, response.Occurrences[index], registry); err != nil {
+			return err
+		}
 		earliest, hasEvidence := canonicalizeItemOccurrence(&response.Occurrences[index], order, sourceID)
 		ordered[index] = orderedItemOccurrenceResponse{
 			value:       response.Occurrences[index],
@@ -40,6 +44,7 @@ func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOr
 	for index := range ordered {
 		response.Occurrences[index] = ordered[index].value
 	}
+	return nil
 }
 
 func canonicalizeItemOccurrence(occurrence *itemOccurrenceResponse, order shared.SourceRefOrder, sourceID string) (int, bool) {
@@ -51,16 +56,23 @@ func canonicalizeItemOccurrence(occurrence *itemOccurrenceResponse, order shared
 	return order.EarliestValid(refs)
 }
 
-func canonicalItemOccurrenceList(response extractionResponse, sourceID string, registry *itemregistry.Registry) dnd.ItemOccurrenceList {
+func validateRegistryPair(index int, occurrence itemOccurrenceResponse, registry *itemregistry.Registry) error {
+	item, found := registry.LookupID(occurrence.ItemID)
+	if !found {
+		return fmt.Errorf("occurrences[%d].item_id is not in the item registry", index)
+	}
+	if item.Name != occurrence.Name {
+		return fmt.Errorf("occurrences[%d].name does not match item_id", index)
+	}
+	return nil
+}
+
+func canonicalItemOccurrenceList(response extractionResponse, sourceID string) dnd.ItemOccurrenceList {
 	if response.Occurrences == nil {
 		return dnd.ItemOccurrenceList{}
 	}
 	occurrences := make([]dnd.ItemOccurrence, 0, len(response.Occurrences))
 	for _, occurrence := range response.Occurrences {
-		item, found := registry.LookupID(occurrence.ItemID)
-		if !found || item.Name != occurrence.Name {
-			continue
-		}
 		occurrences = append(occurrences, dnd.ItemOccurrence{
 			ItemID:     occurrence.ItemID,
 			Name:       occurrence.Name,

internal/modules/dnd/extract/itemoccurrences/extractor.go

@@ -109,6 +109,7 @@ func (e *Extractor) ManifestMetadata() map[string]any {
 		"response_schema_version": SchemaVersion,
 		"response_schema_sha256":  e.responseSchemaSHA,
 		"mapping_policy":          mappingPolicy,
+		"comparison_policy":       shared.TextComparisonPolicy,
 	}
 	seeded := e.itemResolver.Seeded()
 	if seeded.Bound() {
@@ -126,6 +127,7 @@ func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
 		{Name: "prompt", Value: e.promptSHA},
 		{Name: "response_schema", Value: e.responseSchemaSHA},
 		{Name: "mapping_policy", Value: mappingPolicy},
+		{Name: "comparison_policy", Value: shared.TextComparisonPolicy},
 		{Name: "item_registry", Value: e.itemResolver.Seeded().ProjectionDigest()},
 	}
 }
@@ -159,8 +161,10 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
 	}, &response); err != nil {
 		return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
 	}
-	canonicalizeResponse(&response, order, req.Source.ID)
+	if err := canonicalizeResponse(&response, order, req.Source.ID, registry); err != nil {
-	return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: canonicalItemOccurrenceList(response, req.Source.ID, registry)}, nil
+		return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("map item occurrence response: %w", err)
+	}
+	return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: canonicalItemOccurrenceList(response, req.Source.ID)}, nil
 }
 
 func ModuleSpec() pipeline.ModuleSpec {

internal/modules/dnd/extract/itemoccurrences/extractor_test.go

@@ -2,6 +2,7 @@ package itemoccurrences
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -12,8 +13,6 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
 	id := itemidentity.DeriveID("Torch")
 	client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
 		{ItemID: id, Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(1, 1)},
-		{ItemID: "unknown", Name: "Unknown", Kind: "lost", From: "party", SourceRefs: responseRefs(2, 2)},
-		{ItemID: "torch", Name: "Lantern", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)},
 	}}}
 	req := extractionRequest()
 	req.References = itemRegistryReferences(t)
@@ -30,6 +29,56 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
 	}
 }
 
+func TestExtractRejectsInvalidRegistryPairs(t *testing.T) {
+	id := itemidentity.DeriveID("Torch")
+	for _, test := range []struct {
+		name       string
+		occurrence itemOccurrenceResponse
+		wantError  string
+	}{
+		{
+			name:       "unknown item ID",
+			occurrence: itemOccurrenceResponse{ItemID: "unknown", Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(1, 1)},
+			wantError:  "occurrences[0].item_id is not in the item registry",
+		},
+		{
+			name:       "mismatched item name",
+			occurrence: itemOccurrenceResponse{ItemID: id, Name: "Lantern", Kind: "lost", From: "party", SourceRefs: responseRefs(1, 1)},
+			wantError:  "occurrences[0].name does not match item_id",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{test.occurrence}}}
+			req := extractionRequest()
+			req.References = itemRegistryReferences(t)
+			result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
+			if err == nil || !strings.Contains(err.Error(), test.wantError) {
+				t.Fatalf("Extract() error = %v, want %q", err, test.wantError)
+			}
+			if len(result.Value.Occurrences) != 0 {
+				t.Fatalf("Extract() returned partial occurrences: %#v", result.Value.Occurrences)
+			}
+		})
+	}
+}
+
+func TestExtractRejectsResponseWithInvalidRegistryPairAfterValidOccurrence(t *testing.T) {
+	id := itemidentity.DeriveID("Torch")
+	client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
+		{ItemID: id, Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(1, 1)},
+		{ItemID: "unknown", Name: "Unknown", Kind: "lost", From: "party", SourceRefs: responseRefs(2, 2)},
+	}}}
+	req := extractionRequest()
+	req.References = itemRegistryReferences(t)
+	result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
+	if err == nil || !strings.Contains(err.Error(), "occurrences[1].item_id") {
+		t.Fatalf("Extract() error = %v, want occurrence index and item ID", err)
+	}
+	if len(result.Value.Occurrences) != 0 {
+		t.Fatalf("Extract() returned partial occurrences: %#v", result.Value.Occurrences)
+	}
+}
+
 func TestExtractRequiresItemRegistry(t *testing.T) {
 	_, err := newExtractor(t, &fakeItemOccurrencesLLMClient{}).Extract(context.Background(), extractionRequest())
 	if err == nil {

internal/modules/dnd/itemoccurrences/itemoccurrences.go

@@ -8,7 +8,6 @@ import (
 
 	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
-	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
 )
 
@@ -34,7 +33,7 @@ func DisplayValue(value string) string { return strings.TrimSpace(value) }
 
 // ComparisonKey returns the shared D&D Unicode- and case-insensitive key for
 // a trimmed item or holder display value.
-func ComparisonKey(value string) string { return identity.ComparisonKey(DisplayValue(value)) }
+func ComparisonKey(value string) string { return shared.ComparisonKey(DisplayValue(value)) }
 
 // HolderPresent reports whether value is a nonblank optional holder.
 func HolderPresent(value string) bool { return DisplayValue(value) != "" }

internal/modules/dnd/items/identity/identity.go

@@ -8,10 +8,8 @@ import (
 	"fmt"
 	"strings"
 
-	"golang.org/x/text/cases"
-	"golang.org/x/text/unicode/norm"
-
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
 )
 
 const (
@@ -51,17 +49,7 @@ func NormalizeDisplay(value string) string {
 
 // ComparisonKey returns the stable key used for item-type identity comparisons.
 func ComparisonKey(value string) string {
-	value = norm.NFKC.String(value)
+	return shared.ComparisonKey(value)
-	value = strings.Map(func(r rune) rune {
-		switch r {
-		case '\u2018', '\u2019', '\u02bc':
-			return '\''
-		default:
-			return r
-		}
-	}, value)
-	value = strings.Join(strings.Fields(value), " ")
-	return cases.Fold().String(value)
 }
 
 // DeriveID returns the deterministic ID for an item type. Empty identity keys

internal/modules/dnd/items/identity/identity_test.go

@@ -9,29 +9,6 @@ import (
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
 )
 
-func TestComparisonKeyNormalizesSupportedEquivalences(t *testing.T) {
-	tests := []struct {
-		name  string
-		left  string
-		right string
-	}{
-		{name: "case", left: "Silver Key", right: "sILVER kEY"},
-		{name: "compatibility", left: "Ｓｉｌｖｅｒ Ｋｅｙ", right: "Silver Key"},
-		{name: "whitespace", left: "  Silver\u2003Key ", right: "Silver Key"},
-		{name: "apostrophe", left: "Healer’s Kit", right: "healer's kit"},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			if ComparisonKey(test.left) != ComparisonKey(test.right) {
-				t.Fatalf("ComparisonKey(%q) = %q, ComparisonKey(%q) = %q", test.left, ComparisonKey(test.left), test.right, ComparisonKey(test.right))
-			}
-		})
-	}
-	if ComparisonKey("Silver Key") == ComparisonKey("Gold Key") {
-		t.Fatal("different item types received the same comparison key")
-	}
-}
-
 func TestDeriveIDUsesExactCompactIdentityBytes(t *testing.T) {
 	const wantID = "item:sha256:2f211c60b9bdcf3a6dd64086cc4c05df5b3357ae02cb462f7ef8988d9bd2fa4f"
 	const wantIdentity = `["dnd.item_registry.identity.v1","silver key"]`

internal/modules/dnd/locations/identity/identity.go

@@ -10,8 +10,7 @@ import (
 
 	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
-	"golang.org/x/text/cases"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
-	"golang.org/x/text/unicode/norm"
 )
 
 const (
@@ -51,17 +50,7 @@ func NormalizeDisplay(value string) string {
 
 // ComparisonKey returns the stable key used for location identity comparisons.
 func ComparisonKey(value string) string {
-	value = norm.NFKC.String(value)
+	return shared.ComparisonKey(value)
-	value = strings.Map(func(r rune) rune {
-		switch r {
-		case '\u2018', '\u2019', '\u02bc':
-			return '\''
-		default:
-			return r
-		}
-	}, value)
-	value = strings.Join(strings.Fields(value), " ")
-	return cases.Fold().String(value)
 }
 
 // DeriveID derives a location ID from name and the earliest canonical evidence

internal/modules/dnd/locations/identity/identity_test.go

@@ -15,17 +15,6 @@ func TestNormalizeDisplayOnlyChangesWhitespace(t *testing.T) {
 	}
 }
 
-func TestComparisonKeyNormalizesUnicodeWhitespaceAndApostrophes(t *testing.T) {
-	for _, pair := range [][2]string{
-		{"  Caf\u00e9\u2003d\u2019Or  ", "cafe\u0301 d'Or"},
-		{"\uff34\uff48\uff45\u00a0\uff34\uff41\uff56\uff45\uff52\uff4e", "the tavern"},
-	} {
-		if left, right := ComparisonKey(pair[0]), ComparisonKey(pair[1]); left != right {
-			t.Fatalf("ComparisonKey(%q) = %q, want value equal to %q", pair[0], left, pair[1])
-		}
-	}
-}
-
 func TestDeriveIDUsesDocumentedCompactJSONInput(t *testing.T) {
 	refs := []source.SourceRef{
 		{SourceID: "session-alpha", StartUnitID: 9, EndUnitID: 9},

internal/modules/dnd/normalize/itemoccurrences/normalizer.go

@@ -68,7 +68,7 @@ func (n *Normalizer) ManifestMetadata() map[string]any {
 	if n == nil || n.itemResolver == nil {
 		return nil
 	}
-	metadata := map[string]any{"normalization_policy": normalizationPolicy}
+	metadata := map[string]any{"normalization_policy": normalizationPolicy, "comparison_policy": shared.TextComparisonPolicy}
 	seeded := n.itemResolver.Seeded()
 	if seeded.Bound() {
 		metadata["item_registry_digest"] = seeded.Digest()
@@ -83,6 +83,7 @@ func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
 	}
 	return []pipeline.CheckpointFingerprint{
 		{Name: "normalization_policy", Value: normalizationPolicy},
+		{Name: "comparison_policy", Value: shared.TextComparisonPolicy},
 		{Name: "item_registry", Value: n.itemResolver.Seeded().ProjectionDigest()},
 	}
 }

internal/modules/dnd/npcs/identity/identity.go

@@ -9,10 +9,8 @@ import (
 	"fmt"
 	"strings"
 
-	"golang.org/x/text/cases"
-	"golang.org/x/text/unicode/norm"
-
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
 )
 
 const (
@@ -52,17 +50,7 @@ func NormalizeDisplay(value string) string {
 
 // ComparisonKey returns the stable key used for NPC identity comparisons.
 func ComparisonKey(value string) string {
-	value = norm.NFKC.String(value)
+	return shared.ComparisonKey(value)
-	value = strings.Map(func(r rune) rune {
-		switch r {
-		case '\u2018', '\u2019', '\u02bc':
-			return '\''
-		default:
-			return r
-		}
-	}, value)
-	value = strings.Join(strings.Fields(value), " ")
-	return cases.Fold().String(value)
 }
 
 // DeriveID returns the deterministic ID for a canonical NPC name. Empty

internal/modules/dnd/npcs/identity/identity_test.go

@@ -9,30 +9,6 @@ import (
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
 )
 
-func TestComparisonKeyNormalizesSupportedEquivalences(t *testing.T) {
-	tests := []struct {
-		name  string
-		left  string
-		right string
-	}{
-		{name: "case", left: "Captain Vale", right: "cAPtAiN vALE"},
-		{name: "compatibility", left: "Ａｌｌｙ", right: "Ally"},
-		{name: "whitespace", left: "  Mira\u2003Thorn ", right: "Mira Thorn"},
-		{name: "apostrophe", left: "O’Rin", right: "o'Rin"},
-		{name: "modifier apostrophe", left: "OʼRin", right: "o'Rin"},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			if ComparisonKey(test.left) != ComparisonKey(test.right) {
-				t.Fatalf("ComparisonKey(%q) = %q, ComparisonKey(%q) = %q", test.left, ComparisonKey(test.left), test.right, ComparisonKey(test.right))
-			}
-		})
-	}
-	if ComparisonKey("Mira Thorn") == ComparisonKey("Mira Thorne") {
-		t.Fatal("different names received the same comparison key")
-	}
-}
-
 func TestNormalizeDisplayOnlyChangesWhitespace(t *testing.T) {
 	if got := NormalizeDisplay("  O’Rin\u2003Thorn  "); got != "O’Rin Thorn" {
 		t.Fatalf("NormalizeDisplay() = %q", got)

internal/modules/dnd/shared/matching.go

@@ -3,14 +3,12 @@ package shared
 import (
 	"strings"
 	"unicode"
-
-	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
 )
 
 // NormalizedTokens returns identity-normalized alphanumeric tokens from a
 // D&D value.
 func NormalizedTokens(value string) []string {
-	value = identity.ComparisonKey(value)
+	value = ComparisonKey(value)
 	if value == "" {
 		return nil
 	}

internal/modules/dnd/shared/matching_test.go

@@ -2,6 +2,32 @@ package shared
 
 import "testing"
 
+func TestComparisonKeyNormalizesDNDText(t *testing.T) {
+	if TextComparisonPolicy != "dnd.text_comparison.v1" {
+		t.Fatalf("TextComparisonPolicy = %q", TextComparisonPolicy)
+	}
+	for _, test := range []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{name: "case", input: "Captain Vale", want: "captain vale"},
+		{name: "compatibility", input: "Ａｌｌｙ", want: "ally"},
+		{name: "whitespace", input: "  Mira\u2003Thorn ", want: "mira thorn"},
+		{name: "apostrophes", input: "O’Rin and OʼRin", want: "o'rin and o'rin"},
+		{name: "unicode composition", input: "  Cafe\u0301\u2003d’Or ", want: "café d'or"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			if got := ComparisonKey(test.input); got != test.want {
+				t.Fatalf("ComparisonKey(%q) = %q, want %q", test.input, got, test.want)
+			}
+		})
+	}
+	if ComparisonKey("Mira Thorn") == ComparisonKey("Mira Thorne") {
+		t.Fatal("different names received the same comparison key")
+	}
+}
+
 func TestContainsTokenSequence(t *testing.T) {
 	tests := []struct {
 		name  string

internal/modules/dnd/shared/text_comparison.go

@@ -0,0 +1,28 @@
+package shared
+
+import (
+	"strings"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/unicode/norm"
+)
+
+// TextComparisonPolicy identifies the shared D&D text-comparison semantics.
+// A semantic change requires a new value and a review of every dependent
+// identity, mapping, normalization, and validator policy.
+const TextComparisonPolicy = "dnd.text_comparison.v1"
+
+// ComparisonKey returns the D&D Unicode- and case-insensitive comparison key.
+func ComparisonKey(value string) string {
+	value = norm.NFKC.String(value)
+	value = strings.Map(func(r rune) rune {
+		switch r {
+		case '\u2018', '\u2019', '\u02bc':
+			return '\''
+		default:
+			return r
+		}
+	}, value)
+	value = strings.Join(strings.Fields(value), " ")
+	return cases.Fold().String(value)
+}