Collapse duplicate D&D spell casts during normalization

go.mod

@@ -8,4 +8,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require golang.org/x/text v0.40.0 // indirect
+require golang.org/x/text v0.40.0

internal/modules/dnd/normalize/spells/normalizer.go

@@ -4,20 +4,24 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strconv"
+	"strings"
 
 	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
 	spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
+	"golang.org/x/text/cases"
 )
 
 const Key = "dnd/spells"
 
 const (
-	ReasonCodeSpellNameCanonicalized     = "spell_name_canonicalized"
+	ReasonCodeSpellNameCanonicalized      = "spell_name_canonicalized"
-	ReasonCodeSpellNameUnresolved        = "spell_name_unresolved"
+	ReasonCodeSpellNameUnresolved         = "spell_name_unresolved"
-	ReasonCodeSourceReferencesNormalized = "source_references_normalized"
+	ReasonCodeSourceReferencesNormalized  = "source_references_normalized"
+	ReasonCodeDuplicateSpellCastCollapsed = "duplicate_spell_cast_collapsed"
 )
 
 var requiredCapabilities = []string{"merged"}
@@ -86,6 +90,8 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
 	}
 
 	value, warnings := normalizeSpellList(req.MergeOutput.Value, n.effectiveCatalog)
+	value, duplicateWarnings := collapseDuplicateSpellCasts(value, req.Source, n.effectiveCatalog)
+	warnings = append(warnings, duplicateWarnings...)
 	return contracts.TypedNormalizeResult[dnd.SpellList]{Value: value, Warnings: warnings}, nil
 }
 
@@ -179,6 +185,118 @@ func sourceRefLess(left, right source.SourceRef) bool {
 	return left.EndUnitID < right.EndUnitID
 }
 
+type duplicateGroup struct {
+	retainedIndex int
+	removed       []int
+}
+
+func collapseDuplicateSpellCasts(input dnd.SpellList, doc *source.SourceDocument, catalog spellcatalog.EffectiveCatalog) (dnd.SpellList, []contracts.Warning) {
+	if len(input.SpellCasts) == 0 {
+		return input, nil
+	}
+
+	keep := make([]bool, len(input.SpellCasts))
+	groups := make([]duplicateGroup, 0)
+	groupByKey := make(map[string]int)
+	for index, cast := range input.SpellCasts {
+		key, eligible := duplicateKey(cast, doc, catalog)
+		if !eligible {
+			keep[index] = true
+			continue
+		}
+		groupIndex, exists := groupByKey[key]
+		if !exists {
+			groupByKey[key] = len(groups)
+			groups = append(groups, duplicateGroup{retainedIndex: index})
+			keep[index] = true
+			continue
+		}
+		groups[groupIndex].removed = append(groups[groupIndex].removed, index)
+	}
+
+	removedAny := false
+	for _, group := range groups {
+		if len(group.removed) > 0 {
+			removedAny = true
+			break
+		}
+	}
+	if !removedAny {
+		return input, nil
+	}
+
+	output := dnd.SpellList{SpellCasts: make([]dnd.SpellCast, 0, len(input.SpellCasts))}
+	for index, cast := range input.SpellCasts {
+		if keep[index] {
+			output.SpellCasts = append(output.SpellCasts, cast)
+		}
+	}
+
+	warnings := make([]contracts.Warning, 0)
+	for _, group := range groups {
+		if len(group.removed) == 0 {
+			continue
+		}
+		warnings = append(warnings, duplicateWarning(group.retainedIndex, group.removed))
+	}
+	return output, warnings
+}
+
+func duplicateKey(cast dnd.SpellCast, doc *source.SourceDocument, catalog spellcatalog.EffectiveCatalog) (string, bool) {
+	canonicalName, resolved := catalog.Lookup(cast.Spell)
+	if !resolved || len(cast.SourceRefs) == 0 {
+		return "", false
+	}
+	for _, ref := range cast.SourceRefs {
+		if source.ValidateRef(doc, ref) != nil {
+			return "", false
+		}
+	}
+
+	var key strings.Builder
+	writeKeyString(&key, canonicalName)
+	writeKeyString(&key, cases.Fold().String(strings.Join(strings.Fields(cast.Caster), " ")))
+	for _, ref := range cast.SourceRefs {
+		writeKeyString(&key, ref.SourceID)
+		writeKeyInt(&key, ref.StartUnitID)
+		writeKeyInt(&key, ref.EndUnitID)
+	}
+	return key.String(), true
+}
+
+func writeKeyString(builder *strings.Builder, value string) {
+	builder.WriteString(strconv.Itoa(len(value)))
+	builder.WriteByte(':')
+	builder.WriteString(value)
+}
+
+func writeKeyInt(builder *strings.Builder, value int) {
+	builder.WriteString(strconv.Itoa(value))
+	builder.WriteByte(';')
+}
+
+func duplicateWarning(retainedIndex int, removed []int) contracts.Warning {
+	const maxDisplayedIndices = 20
+	displayed := removed
+	if len(displayed) > maxDisplayedIndices {
+		displayed = displayed[:maxDisplayedIndices]
+	}
+	indices := make([]string, len(displayed))
+	for index, removedIndex := range displayed {
+		indices[index] = strconv.Itoa(removedIndex)
+	}
+
+	message := fmt.Sprintf("retained input index %d; removed input indices [%s]", retainedIndex, strings.Join(indices, ", "))
+	if omitted := len(removed) - len(displayed); omitted > 0 {
+		message += fmt.Sprintf("; %d additional removed input indices omitted", omitted)
+	}
+	return contracts.Warning{
+		Scope:      spellCastScope(retainedIndex),
+		ReasonCode: ReasonCodeDuplicateSpellCastCollapsed,
+		Message:    message,
+	}
+}
+
 func boundedName(name string) string {
 	runes := []rune(name)
 	if len(runes) <= 128 {

internal/modules/dnd/normalize/spells/normalizer_test.go

@@ -260,6 +260,176 @@ func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) {
 	}
 }
 
+func TestNormalizeCollapsesDuplicateGroupsAfterCanonicalization(t *testing.T) {
+	normalizer := newNormalizer(t)
+	doc := sourceDocument(6)
+	firstEvidence := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 2}
+	secondEvidence := source.SourceRef{SourceID: "source", StartUnitID: 3, EndUnitID: 4}
+	input := dnd.SpellList{SpellCasts: []dnd.SpellCast{
+		{Caster: "  Aria \t", Spell: " cure   wounds ", Effect: "first effect", NarrativeDescription: "first narrative", SourceRefs: []source.SourceRef{secondEvidence, firstEvidence, firstEvidence}},
+		{Caster: "Borin", Spell: "Healing Word", Effect: "distinct effect", NarrativeDescription: "distinct narrative", SourceRefs: []source.SourceRef{firstEvidence}},
+		{Caster: " Kyle ", Spell: "Cure Wounds", Effect: "kept effect", NarrativeDescription: "kept narrative", SourceRefs: []source.SourceRef{firstEvidence}},
+		{Caster: "aria", Spell: " cure wounds ", Effect: "removed effect", NarrativeDescription: "removed narrative", SourceRefs: []source.SourceRef{firstEvidence, secondEvidence}},
+		{Caster: "KYLE", Spell: " cure wounds ", Effect: "removed effect two", NarrativeDescription: "removed narrative two", SourceRefs: []source.SourceRef{firstEvidence}},
+		{Caster: " kyle ", Spell: "Cure Wounds", Effect: "removed effect three", NarrativeDescription: "removed narrative three", SourceRefs: []source.SourceRef{firstEvidence}},
+	}}
+
+	result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
+	if err != nil {
+		t.Fatalf("Normalize() error = %v, want nil", err)
+	}
+	if len(result.Value.SpellCasts) != 3 {
+		t.Fatalf("normalized casts = %#v, want first occurrences plus distinct cast", result.Value.SpellCasts)
+	}
+	if got := result.Value.SpellCasts[0]; got.Caster != "  Aria \t" || got.Effect != "first effect" || got.NarrativeDescription != "first narrative" {
+		t.Fatalf("retained first cast = %#v, want first occurrence fields unchanged", got)
+	}
+	if got := result.Value.SpellCasts[0].SourceRefs; !reflect.DeepEqual(got, []source.SourceRef{firstEvidence, secondEvidence}) {
+		t.Fatalf("retained first evidence = %#v, want canonical first evidence only", got)
+	}
+	if got := result.Value.SpellCasts[2].Caster; got != " Kyle " {
+		t.Fatalf("Unicode caster output = %q, want first occurrence text unchanged", got)
+	}
+
+	if len(result.Warnings) != 6 {
+		t.Fatalf("warnings = %#v, want per-cast warnings followed by two group warnings", result.Warnings)
+	}
+	if result.Warnings[0].ReasonCode != ReasonCodeSpellNameCanonicalized || result.Warnings[0].Scope != "spell_casts[0]" {
+		t.Fatalf("warning[0] = %#v, want input name warning", result.Warnings[0])
+	}
+	if result.Warnings[1].ReasonCode != ReasonCodeSourceReferencesNormalized || result.Warnings[1].Scope != "spell_casts[0]" {
+		t.Fatalf("warning[1] = %#v, want input source warning", result.Warnings[1])
+	}
+	if result.Warnings[2].ReasonCode != ReasonCodeSpellNameCanonicalized || result.Warnings[2].Scope != "spell_casts[3]" {
+		t.Fatalf("warning[2] = %#v, want removed occurrence warning", result.Warnings[2])
+	}
+	if result.Warnings[3].ReasonCode != ReasonCodeSpellNameCanonicalized || result.Warnings[3].Scope != "spell_casts[4]" {
+		t.Fatalf("warning[3] = %#v, want removed occurrence warning", result.Warnings[3])
+	}
+	if result.Warnings[4].ReasonCode != ReasonCodeDuplicateSpellCastCollapsed || result.Warnings[4].Scope != "spell_casts[0]" || !strings.Contains(result.Warnings[4].Message, "retained input index 0") || !strings.Contains(result.Warnings[4].Message, "removed input indices [3]") {
+		t.Fatalf("warning[4] = %#v, want first duplicate group warning", result.Warnings[4])
+	}
+	if result.Warnings[5].ReasonCode != ReasonCodeDuplicateSpellCastCollapsed || result.Warnings[5].Scope != "spell_casts[2]" || !strings.Contains(result.Warnings[5].Message, "removed input indices [4, 5]") {
+		t.Fatalf("warning[5] = %#v, want second duplicate group warning", result.Warnings[5])
+	}
+}
+
+func TestNormalizeKeepsDistinctAndIneligibleCastsSeparate(t *testing.T) {
+	doc := sourceDocument(6)
+	ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 2}
+	otherRef := source.SourceRef{SourceID: "source", StartUnitID: 3, EndUnitID: 4}
+	tests := []struct {
+		name    string
+		variant dnd.SpellCast
+	}{
+		{name: "distinct canonical spell", variant: dnd.SpellCast{Spell: "Healing Word", Caster: "Aria", SourceRefs: []source.SourceRef{ref}}},
+		{name: "distinct caster", variant: dnd.SpellCast{Spell: "Cure Wounds", Caster: "Borin", SourceRefs: []source.SourceRef{ref}}},
+		{name: "distinct evidence", variant: dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{otherRef}}},
+		{name: "unknown spell", variant: dnd.SpellCast{Spell: "Unknown Spell", Caster: "Aria", SourceRefs: []source.SourceRef{ref}}},
+		{name: "empty evidence", variant: dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria"}},
+		{name: "invalid evidence", variant: dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 2}}}},
+	}
+	base := dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{ref}}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(dnd.SpellList{SpellCasts: []dnd.SpellCast{base, test.variant}}, doc))
+			if err != nil {
+				t.Fatalf("Normalize() error = %v, want nil", err)
+			}
+			if len(result.Value.SpellCasts) != 2 {
+				t.Fatalf("normalized casts = %#v, want both casts retained", result.Value.SpellCasts)
+			}
+			for _, warning := range result.Warnings {
+				if warning.ReasonCode == ReasonCodeDuplicateSpellCastCollapsed {
+					t.Fatalf("warnings = %#v, want no duplicate collapse", result.Warnings)
+				}
+			}
+		})
+	}
+}
+
+func TestNormalizeDoesNotCollapseAdjacentOrOverlappingEvidence(t *testing.T) {
+	doc := sourceDocument(8)
+	input := dnd.SpellList{SpellCasts: []dnd.SpellCast{
+		{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 2}}},
+		{Spell: "Cure Wounds", Caster: "aria", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 3, EndUnitID: 4}}},
+		{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 3}}},
+		{Spell: "Cure Wounds", Caster: "aria", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 2, EndUnitID: 4}}},
+	}}
+	result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
+	if err != nil {
+		t.Fatalf("Normalize() error = %v, want nil", err)
+	}
+	if len(result.Value.SpellCasts) != len(input.SpellCasts) {
+		t.Fatalf("normalized casts = %#v, want adjacent and overlapping evidence retained", result.Value.SpellCasts)
+	}
+	for _, warning := range result.Warnings {
+		if warning.ReasonCode == ReasonCodeDuplicateSpellCastCollapsed {
+			t.Fatalf("warnings = %#v, want no duplicate collapse", result.Warnings)
+		}
+	}
+}
+
+func TestNormalizeBoundsDuplicateWarningIndices(t *testing.T) {
+	doc := sourceDocument(2)
+	ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}
+	casts := make([]dnd.SpellCast, 22)
+	for index := range casts {
+		casts[index] = dnd.SpellCast{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{ref}}
+	}
+	result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(dnd.SpellList{SpellCasts: casts}, doc))
+	if err != nil {
+		t.Fatalf("Normalize() error = %v, want nil", err)
+	}
+	if len(result.Value.SpellCasts) != 1 || len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != ReasonCodeDuplicateSpellCastCollapsed {
+		t.Fatalf("result = %#v, warnings = %#v, want one retained cast and one bounded warning", result.Value, result.Warnings)
+	}
+	message := result.Warnings[0].Message
+	if !strings.Contains(message, "removed input indices [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]") || strings.Contains(message, ", 21]") || !strings.Contains(message, "1 additional removed input indices omitted") {
+		t.Fatalf("warning message = %q, want 20 displayed indices and exact omitted count", message)
+	}
+}
+
+func TestNormalizeIsIdempotentForAlreadyNormalizedInput(t *testing.T) {
+	doc := sourceDocument(3)
+	input := dnd.SpellList{SpellCasts: []dnd.SpellCast{{
+		Spell:                "Cure Wounds",
+		Caster:               "Aria",
+		Effect:               "effect",
+		NarrativeDescription: "narrative",
+		SourceRefs:           []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 2}},
+	}}}
+	normalizer := newNormalizer(t)
+	first, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc))
+	if err != nil {
+		t.Fatalf("first Normalize() error = %v, want nil", err)
+	}
+	second, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
+	if err != nil {
+		t.Fatalf("second Normalize() error = %v, want nil", err)
+	}
+	if !reflect.DeepEqual(second.Value, first.Value) || len(first.Warnings) != 0 || len(second.Warnings) != 0 {
+		t.Fatalf("first = %#v/%#v, second = %#v/%#v, want identical artifacts without mutation warnings", first.Value, first.Warnings, second.Value, second.Warnings)
+	}
+}
+
+func TestNormalizeDuplicateOutputDoesNotShareInputSlices(t *testing.T) {
+	doc := sourceDocument(3)
+	ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 2}
+	input := dnd.SpellList{SpellCasts: []dnd.SpellCast{
+		{Spell: "Cure Wounds", Caster: "Aria", SourceRefs: []source.SourceRef{ref}},
+		{Spell: "Cure Wounds", Caster: "aria", SourceRefs: []source.SourceRef{ref}},
+	}}
+	result, err := newNormalizer(t).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
+	if err != nil {
+		t.Fatalf("Normalize() error = %v, want nil", err)
+	}
+	result.Value.SpellCasts[0].SourceRefs[0].SourceID = "changed"
+	if input.SpellCasts[0].SourceRefs[0].SourceID != "source" || input.SpellCasts[1].SourceRefs[0].SourceID != "source" {
+		t.Fatalf("normalized output shares input references: input = %#v", input)
+	}
+}
+
 func newNormalizer(t *testing.T) *Normalizer {
 	t.Helper()
 	normalizer, err := New(Options{}, overlayReference())
@@ -281,6 +451,20 @@ func normalizeRequest(value dnd.SpellList) contracts.TypedNormalizeRequest[dnd.S
 	}
 }
 
+func normalizeRequestWithSource(value dnd.SpellList, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.SpellList] {
+	request := normalizeRequest(value)
+	request.Source = doc
+	return request
+}
+
+func sourceDocument(unitCount int) *source.SourceDocument {
+	units := make([]source.SourceUnit, unitCount)
+	for index := range units {
+		units[index] = source.SourceUnit{ID: index + 1}
+	}
+	return &source.SourceDocument{ID: "source", Units: units}
+}
+
 func overlayReference() contracts.ReferenceSet {
 	return spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v1","catalogs":[{"id":"campaign.example","ruleset":"dnd-5e-2014","source":{"title":"Private campaign source","version":"1","url":"file:///private-source.json","license":"private"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]}]}`)
 }