Add NPC normalization and identity validation

2026-07-21 02:36:59 +00:00
parent 3d5fd9dc05
commit 06c0259788
4 changed files with 1014 additions and 0 deletions
--- a/internal/modules/dnd/normalize/npcs/normalizer.go
+++ b/internal/modules/dnd/normalize/npcs/normalizer.go
@@ -0,0 +1,546 @@
+// Package npcs normalizes merged D&D non-player character records.
+package npcs
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/diagnostics"
+)
+
+const (
+	Key                 = "dnd/npcs"
+	NormalizationPolicy = "dnd.npcs.normalize.v1"
+
+	ReasonCodeNPCFieldsNormalized             = "npc_fields_normalized"
+	ReasonCodeNPCIDRecomputed                 = "npc_id_recomputed"
+	ReasonCodeSourceReferencesNormalized      = "source_references_normalized"
+	ReasonCodeDuplicateNPCCollapsed           = "duplicate_npc_collapsed"
+	ReasonCodeRelationshipTargetCanonicalized = "relationship_target_canonicalized"
+)
+
+var requiredCapabilities = []string{"merged"}
+var providedCapabilities = []string{"normalized"}
+
+var _ contracts.Normalizer[dnd.NPCList] = (*Normalizer)(nil)
+var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil)
+var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil)
+
+type Options struct{}
+
+type Normalizer struct{}
+
+func New(Options) *Normalizer { return &Normalizer{} }
+
+func (n *Normalizer) Key() string { return Key }
+
+func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
+
+func (n *Normalizer) ManifestMetadata() map[string]any {
+	if n == nil {
+		return nil
+	}
+	return map[string]any{
+		"identity_policy":      identity.Policy,
+		"normalization_policy": NormalizationPolicy,
+	}
+}
+
+func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
+	if n == nil {
+		return nil
+	}
+	return []pipeline.CheckpointFingerprint{
+		{Name: "identity_policy", Value: identity.Policy},
+		{Name: "normalization_policy", Value: NormalizationPolicy},
+	}
+}
+
+func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.NPCList]) (contracts.TypedNormalizeResult[dnd.NPCList], error) {
+	if n == nil {
+		return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("normalizer must not be nil")
+	}
+	if ctx == nil {
+		return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("context must not be nil")
+	}
+	if err := ctx.Err(); err != nil {
+		return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("context error before normalize: %w", err)
+	}
+
+	value, warnings := normalizeList(req.MergeOutput.Value)
+	return contracts.TypedNormalizeResult[dnd.NPCList]{Value: value, Warnings: warnings}, nil
+}
+
+type normalizedRecord struct {
+	npc               dnd.NPC
+	fieldsChanged     bool
+	referencesChanged bool
+}
+
+func normalizeList(input dnd.NPCList) (dnd.NPCList, []contracts.Warning) {
+	if input.NPCs == nil {
+		return dnd.NPCList{}, nil
+	}
+
+	records := make([]normalizedRecord, len(input.NPCs))
+	warnings := make([]contracts.Warning, 0)
+	for index, inputNPC := range input.NPCs {
+		npc, fieldsChanged, referencesChanged := normalizeRecord(inputNPC)
+		records[index] = normalizedRecord{
+			npc:               npc,
+			fieldsChanged:     fieldsChanged,
+			referencesChanged: referencesChanged,
+		}
+		if fieldsChanged {
+			warnings = append(warnings, contracts.Warning{
+				Scope:      npcScope(index),
+				ReasonCode: ReasonCodeNPCFieldsNormalized,
+				Message: fmt.Sprintf("input index %d: NPC fields normalized for %s",
+					index, diagnostics.Quote(inputNPC.Name)),
+			})
+		}
+		if referencesChanged {
+			warnings = append(warnings, contracts.Warning{
+				Scope:      npcScope(index),
+				ReasonCode: ReasonCodeSourceReferencesNormalized,
+				Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)",
+					index, len(inputNPC.SourceRefs), len(npc.SourceRefs)),
+			})
+		}
+		if inputNPC.ID != npc.ID {
+			warnings = append(warnings, contracts.Warning{
+				Scope:      npcScope(index),
+				ReasonCode: ReasonCodeNPCIDRecomputed,
+				Message: fmt.Sprintf("input index %d: NPC ID recomputed from %s",
+					index, diagnostics.Quote(npc.Name)),
+			})
+		}
+	}
+
+	components := identityComponents(records)
+	output := dnd.NPCList{NPCs: make([]dnd.NPC, 0, len(components))}
+	retainedIndexes := make([]int, 0, len(components))
+	for _, members := range components {
+		consolidated, sourceChanged := consolidate(records, members)
+		retainedIndex := members[0]
+		output.NPCs = append(output.NPCs, consolidated)
+		retainedIndexes = append(retainedIndexes, retainedIndex)
+
+		if sourceChanged {
+			warnings = append(warnings, contracts.Warning{
+				Scope:      npcScope(retainedIndex),
+				ReasonCode: ReasonCodeSourceReferencesNormalized,
+				Message: fmt.Sprintf("input index %d: source references normalized during identity consolidation (final count %d)",
+					retainedIndex, len(consolidated.SourceRefs)),
+			})
+		}
+		if len(members) > 1 {
+			warnings = append(warnings, duplicateWarning(retainedIndex, members[1:]))
+		}
+	}
+
+	warnings = append(warnings, canonicalizeRelationshipTargets(&output, retainedIndexes)...)
+	return output, warnings
+}
+
+func normalizeRecord(input dnd.NPC) (dnd.NPC, bool, bool) {
+	output := cloneNPC(input)
+	output.Name = identity.NormalizeDisplay(input.Name)
+	output.Aliases = normalizeAliases(input.Aliases, output.Name)
+	output.Description = strings.TrimSpace(input.Description)
+	output.Relationships = normalizeRelationships(input.Relationships)
+	output.SourceRefs, _, _ = canonicalizeSourceRefs(input.SourceRefs)
+	output.ID = identity.DeriveID(output.Name)
+
+	fieldsChanged := input.Name != output.Name ||
+		!reflect.DeepEqual(input.Aliases, output.Aliases) ||
+		input.Description != output.Description ||
+		!reflect.DeepEqual(input.Relationships, output.Relationships)
+	referencesChanged := !reflect.DeepEqual(input.SourceRefs, output.SourceRefs)
+	return output, fieldsChanged, referencesChanged
+}
+
+func cloneNPC(input dnd.NPC) dnd.NPC {
+	output := input
+	if input.Aliases != nil {
+		output.Aliases = append([]string(nil), input.Aliases...)
+	}
+	if input.Relationships != nil {
+		output.Relationships = make([]dnd.NPCRelationship, len(input.Relationships))
+		copy(output.Relationships, input.Relationships)
+	}
+	if input.SourceRefs != nil {
+		output.SourceRefs = make([]source.SourceRef, len(input.SourceRefs))
+		copy(output.SourceRefs, input.SourceRefs)
+	}
+	return output
+}
+
+func normalizeAliases(input []string, canonicalName string) []string {
+	if input == nil {
+		return nil
+	}
+
+	canonicalKey := identity.ComparisonKey(canonicalName)
+	output := make([]string, 0, len(input))
+	seen := make(map[string]struct{}, len(input))
+	for _, alias := range input {
+		normalized := identity.NormalizeDisplay(alias)
+		key := identity.ComparisonKey(normalized)
+		if canonicalKey != "" && key == canonicalKey {
+			continue
+		}
+		if _, exists := seen[key]; exists {
+			continue
+		}
+		seen[key] = struct{}{}
+		output = append(output, normalized)
+	}
+	return output
+}
+
+func normalizeRelationships(input []dnd.NPCRelationship) []dnd.NPCRelationship {
+	if input == nil {
+		return nil
+	}
+
+	output := make([]dnd.NPCRelationship, 0, len(input))
+	seen := make(map[relationshipIdentity]struct{}, len(input))
+	for _, relationship := range input {
+		normalized := dnd.NPCRelationship{
+			Target:       identity.NormalizeDisplay(relationship.Target),
+			Relationship: strings.TrimSpace(relationship.Relationship),
+		}
+		key := relationshipKey(normalized)
+		if _, exists := seen[key]; exists {
+			continue
+		}
+		seen[key] = struct{}{}
+		output = append(output, normalized)
+	}
+	return output
+}
+
+type relationshipIdentity struct {
+	target       string
+	relationship string
+}
+
+func relationshipKey(relationship dnd.NPCRelationship) relationshipIdentity {
+	return relationshipIdentity{
+		target:       identity.ComparisonKey(relationship.Target),
+		relationship: identity.ComparisonKey(relationship.Relationship),
+	}
+}
+
+func canonicalizeSourceRefs(input []source.SourceRef) ([]source.SourceRef, bool, int) {
+	if input == nil {
+		return nil, false, 0
+	}
+
+	canonical := make([]source.SourceRef, len(input))
+	copy(canonical, input)
+	sort.SliceStable(canonical, func(left, right int) bool {
+		return sourceRefLess(canonical[left], canonical[right])
+	})
+
+	orderChanged := false
+	for index := range input {
+		if input[index] != canonical[index] {
+			orderChanged = true
+			break
+		}
+	}
+
+	unique := make([]source.SourceRef, 0, len(canonical))
+	for _, ref := range canonical {
+		if len(unique) == 0 || unique[len(unique)-1] != ref {
+			unique = append(unique, ref)
+		}
+	}
+	return unique, orderChanged, len(input) - len(unique)
+}
+
+func sourceRefLess(left, right source.SourceRef) bool {
+	if left.SourceID != right.SourceID {
+		return left.SourceID < right.SourceID
+	}
+	if left.StartUnitID != right.StartUnitID {
+		return left.StartUnitID < right.StartUnitID
+	}
+	return left.EndUnitID < right.EndUnitID
+}
+
+func identityComponents(records []normalizedRecord) [][]int {
+	parent := make([]int, len(records))
+	for index := range parent {
+		parent[index] = index
+	}
+
+	for left := 0; left < len(records); left++ {
+		for right := left + 1; right < len(records); right++ {
+			if recordsCanMerge(records[left].npc, records[right].npc) {
+				union(parent, left, right)
+			}
+		}
+	}
+
+	byRoot := make(map[int][]int, len(records))
+	for index := range records {
+		root := find(parent, index)
+		byRoot[root] = append(byRoot[root], index)
+	}
+	roots := make([]int, 0, len(byRoot))
+	for root := range byRoot {
+		roots = append(roots, root)
+	}
+	sort.Slice(roots, func(left, right int) bool {
+		return byRoot[roots[left]][0] < byRoot[roots[right]][0]
+	})
+
+	components := make([][]int, 0, len(roots))
+	for _, root := range roots {
+		components = append(components, byRoot[root])
+	}
+	return components
+}
+
+func recordsCanMerge(left, right dnd.NPC) bool {
+	leftCanonical := identity.ComparisonKey(left.Name)
+	rightCanonical := identity.ComparisonKey(right.Name)
+	return leftCanonical == rightCanonical ||
+		containsAliasKey(right.Aliases, leftCanonical) ||
+		containsAliasKey(left.Aliases, rightCanonical)
+}
+
+func containsAliasKey(aliases []string, wanted string) bool {
+	for _, alias := range aliases {
+		if identity.ComparisonKey(alias) == wanted {
+			return true
+		}
+	}
+	return false
+}
+
+func find(parent []int, index int) int {
+	for parent[index] != index {
+		parent[index] = parent[parent[index]]
+		index = parent[index]
+	}
+	return index
+}
+
+func union(parent []int, left, right int) {
+	leftRoot := find(parent, left)
+	rightRoot := find(parent, right)
+	if leftRoot == rightRoot {
+		return
+	}
+	if leftRoot < rightRoot {
+		parent[rightRoot] = leftRoot
+	} else {
+		parent[leftRoot] = rightRoot
+	}
+}
+
+func consolidate(records []normalizedRecord, members []int) (dnd.NPC, bool) {
+	output := cloneNPC(records[members[0]].npc)
+	originalRefs := cloneSourceRefs(output.SourceRefs)
+	canonicalKey := identity.ComparisonKey(output.Name)
+
+	for _, member := range members[1:] {
+		candidate := records[member].npc
+		appendAlias(&output.Aliases, candidate.Name, canonicalKey)
+		for _, alias := range candidate.Aliases {
+			appendAlias(&output.Aliases, alias, canonicalKey)
+		}
+		for _, relationship := range candidate.Relationships {
+			appendRelationship(&output.Relationships, relationship)
+		}
+		output.SourceRefs = append(output.SourceRefs, candidate.SourceRefs...)
+	}
+
+	output.SourceRefs, _, _ = canonicalizeSourceRefs(output.SourceRefs)
+	output.ID = identity.DeriveID(output.Name)
+	return output, !reflect.DeepEqual(originalRefs, output.SourceRefs)
+}
+
+func cloneSourceRefs(input []source.SourceRef) []source.SourceRef {
+	if input == nil {
+		return nil
+	}
+	output := make([]source.SourceRef, len(input))
+	copy(output, input)
+	return output
+}
+
+func appendAlias(aliases *[]string, value, canonicalKey string) {
+	key := identity.ComparisonKey(value)
+	if canonicalKey != "" && key == canonicalKey {
+		return
+	}
+	for _, existing := range *aliases {
+		if identity.ComparisonKey(existing) == key {
+			return
+		}
+	}
+	*aliases = append(*aliases, value)
+}
+
+func appendRelationship(relationships *[]dnd.NPCRelationship, relationship dnd.NPCRelationship) {
+	key := relationshipKey(relationship)
+	for _, existing := range *relationships {
+		if relationshipKey(existing) == key {
+			return
+		}
+	}
+	*relationships = append(*relationships, relationship)
+}
+
+func canonicalizeRelationshipTargets(list *dnd.NPCList, retainedIndexes []int) []contracts.Warning {
+	if list == nil || len(list.NPCs) == 0 {
+		return nil
+	}
+
+	owners := make(map[string][]int)
+	for index, npc := range list.NPCs {
+		if key := identity.ComparisonKey(npc.Name); key != "" {
+			owners[key] = append(owners[key], index)
+		}
+		for _, alias := range npc.Aliases {
+			if key := identity.ComparisonKey(alias); key != "" {
+				owners[key] = appendUniqueIndex(owners[key], index)
+			}
+		}
+	}
+
+	warnings := make([]contracts.Warning, 0)
+	for outputIndex := range list.NPCs {
+		npc := &list.NPCs[outputIndex]
+		originalRelationshipCount := len(npc.Relationships)
+		for relationshipIndex := range npc.Relationships {
+			relationship := &npc.Relationships[relationshipIndex]
+			key := identity.ComparisonKey(relationship.Target)
+			if key == "" || len(owners[key]) != 1 {
+				continue
+			}
+			target := list.NPCs[owners[key][0]].Name
+			if relationship.Target == target {
+				continue
+			}
+			oldTarget := relationship.Target
+			relationship.Target = target
+			warnings = append(warnings, contracts.Warning{
+				Scope:      npcScope(retainedIndexes[outputIndex]),
+				ReasonCode: ReasonCodeRelationshipTargetCanonicalized,
+				Message: fmt.Sprintf("input index %d: relationship target canonicalized from %s to %s",
+					retainedIndexes[outputIndex], diagnostics.Quote(oldTarget), diagnostics.Quote(target)),
+			})
+		}
+		npc.Relationships = deduplicateRelationships(npc.Relationships)
+		if len(npc.Relationships) != originalRelationshipCount {
+			warnings = append(warnings, contracts.Warning{
+				Scope:      npcScope(retainedIndexes[outputIndex]),
+				ReasonCode: ReasonCodeNPCFieldsNormalized,
+				Message: fmt.Sprintf("input index %d: duplicate relationships removed after target canonicalization",
+					retainedIndexes[outputIndex]),
+			})
+		}
+	}
+	return warnings
+}
+
+func deduplicateRelationships(input []dnd.NPCRelationship) []dnd.NPCRelationship {
+	if input == nil {
+		return nil
+	}
+	output := make([]dnd.NPCRelationship, 0, len(input))
+	seen := make(map[relationshipIdentity]struct{}, len(input))
+	for _, relationship := range input {
+		key := relationshipKey(relationship)
+		if _, exists := seen[key]; exists {
+			continue
+		}
+		seen[key] = struct{}{}
+		output = append(output, relationship)
+	}
+	return output
+}
+
+func appendUniqueIndex(values []int, wanted int) []int {
+	for _, value := range values {
+		if value == wanted {
+			return values
+		}
+	}
+	return append(values, wanted)
+}
+
+func duplicateWarning(retainedIndex int, removed []int) contracts.Warning {
+	const maxDisplayedIndices = 20
+	displayed := removed
+	if len(displayed) > maxDisplayedIndices {
+		displayed = displayed[:maxDisplayedIndices]
+	}
+	indices := make([]string, len(displayed))
+	for index, removedIndex := range displayed {
+		indices[index] = strconv.Itoa(removedIndex)
+	}
+
+	message := fmt.Sprintf("retained input index %d; removed input indices [%s]", retainedIndex, strings.Join(indices, ", "))
+	if omitted := len(removed) - len(displayed); omitted > 0 {
+		message += fmt.Sprintf("; %d additional removed input indices omitted", omitted)
+	}
+	return contracts.Warning{
+		Scope:      npcScope(retainedIndex),
+		ReasonCode: ReasonCodeDuplicateNPCCollapsed,
+		Message:    message,
+	}
+}
+
+func npcScope(index int) string { return fmt.Sprintf("npcs[%d]", index) }
+
+func ModuleSpec() pipeline.ModuleSpec {
+	return pipeline.ModuleSpec{
+		Key:          Key,
+		Stage:        pipeline.StageNormalize,
+		Requires:     append([]string(nil), requiredCapabilities...),
+		Provides:     append([]string(nil), providedCapabilities...),
+		ArtifactKind: dnd.NPCListKind,
+	}
+}
+
+func Register(registry *pipeline.NormalizerRegistry) error {
+	return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.NPCList], error) {
+		options, err := DecodeOptions(request.Options)
+		if err != nil {
+			return nil, err
+		}
+		return New(options), nil
+	})
+}
+
+func validateOptions(options map[string]any) error {
+	_, err := DecodeOptions(options)
+	return err
+}
+
+func DecodeOptions(options map[string]any) (Options, error) {
+	if err := pipeline.RejectUnknownOptions(options); err != nil {
+		return Options{}, normalizerErrorf("%w", err)
+	}
+	return Options{}, nil
+}
+
+func normalizerErrorf(format string, args ...any) error {
+	return fmt.Errorf("dnd npcs normalizer: "+format, args...)
+}