eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Consolidate duplicate item registry records

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-06 13:28:12 +00:00
parent f1b120b590
commit 2ee6b495e1
3 changed files with 355 additions and 802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1096
docs/roadmap/implementation.md
View File

File diff suppressed because it is too large Load Diff

9
internal/modules/dnd/normalize/itemregistry/normalizer.go
View File

@@ -212,13 +212,16 @@ func preprocessRecords(input dnd.ItemRegistry, order shared.SourceRefOrder) ([]n
warnings = append(warnings, contracts.Warning{Scope: itemScope(index), ReasonCode: ReasonCodeItemIDRecomputed, Message: fmt.Sprintf("input index %d: item ID recomputed from %s", index, diagnostics.Quote(item.Name))}) warnings = append(warnings, contracts.Warning{Scope: itemScope(index), ReasonCode: ReasonCodeItemIDRecomputed, Message: fmt.Sprintf("input index %d: item ID recomputed from %s", index, diagnostics.Quote(item.Name))})
} }
} }
groups := exactDuplicateGroups(records) groups := comparisonNameGroups(records)
output := make([]normalizedRecord, 0, len(groups)) output := make([]normalizedRecord, 0, len(groups))
for _, members := range groups { for _, members := range groups {
retained := cloneRecord(records[members[0]]) retained := cloneRecord(records[members[0]])
for _, member := range members[1:] { for _, member := range members[1:] {
retained.item.SourceRefs = append(retained.item.SourceRefs, records[member].item.SourceRefs...)
retained.inputIndexes = append(retained.inputIndexes, records[member].inputIndexes...) retained.inputIndexes = append(retained.inputIndexes, records[member].inputIndexes...)
} }
retained.item.SourceRefs = order.Canonicalize(retained.item.SourceRefs)
retained.item.ID = identity.DeriveID(retained.item.Name)
retained.inputIndexes = sortedUniqueIndexes(retained.inputIndexes) retained.inputIndexes = sortedUniqueIndexes(retained.inputIndexes)
output = append(output, retained) output = append(output, retained)
if len(members) > 1 { if len(members) > 1 {
@@ -236,14 +239,14 @@ func normalizeRecord(input dnd.Item, order shared.SourceRefOrder) (dnd.Item, boo
return output, input.Name != output.Name, !reflect.DeepEqual(input.SourceRefs, output.SourceRefs) return output, input.Name != output.Name, !reflect.DeepEqual(input.SourceRefs, output.SourceRefs)
} }
func exactDuplicateGroups(records []normalizedRecord) [][]int { func comparisonNameGroups(records []normalizedRecord) [][]int {
groups := make([][]int, 0, len(records)) groups := make([][]int, 0, len(records))
for index, record := range records { for index, record := range records {
key := identity.ComparisonKey(record.item.Name) key := identity.ComparisonKey(record.item.Name)
found := false found := false
for groupIndex, members := range groups { for groupIndex, members := range groups {
first := records[members[0]] first := records[members[0]]
if identity.ComparisonKey(first.item.Name) == key && reflect.DeepEqual(first.item.SourceRefs, record.item.SourceRefs) { if identity.ComparisonKey(first.item.Name) == key {
groups[groupIndex] = append(groups[groupIndex], index) groups[groupIndex] = append(groups[groupIndex], index)
found = true found = true
break break

52
internal/modules/dnd/normalize/itemregistry/normalizer_test.go
View File

@@ -17,6 +17,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile"
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
"gitea.maximumdirect.net/eric/promptkit" "gitea.maximumdirect.net/eric/promptkit"
) )
@@ -38,19 +39,48 @@ func TestModuleContractAndMetadata(t *testing.T) {
} }
} }
func TestNormalizePreprocessesExactDuplicatesWithoutMutation(t *testing.T) { func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) {
input := dnd.ItemRegistry{Items: []dnd.Item{ doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}, {ID: 1, Text: "The rope is secured."},
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}, {ID: 2, Text: "The party takes the rope."},
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}}}, {ID: 3, Text: "The rope is packed away."},
{ID: 4, Text: "A lantern lights the path."},
}} }}
before := dnd.ItemRegistry{Items: append([]dnd.Item(nil), input.Items...)} input := dnd.ItemRegistry{Items: []dnd.Item{
result, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input)) {Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) { {Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
t.Fatalf("Normalize() = %#v, %v; want non-mutating exact deduplication", result, err) {Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
}}
before := dnd.ItemRegistry{Items: []dnd.Item{
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
}}
client := &recordingNormalizerClient{}
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) || len(client.requests) != 1 {
t.Fatalf("Normalize() = %#v, %v; want deterministic non-mutating consolidation", result, err)
} }
if result.Value.Items[0].ID != result.Value.Items[1].ID || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) { rope := result.Value.Items[0]
t.Fatalf("items = %#v, warnings = %#v; want type identity and duplicate warning", result.Value.Items, result.Warnings) wantRefs := []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}
if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
t.Fatalf("items = %#v, warnings = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Warnings)
}
validation, validationErr := identityvalidator.New(identityvalidator.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: result.Value})
if validationErr != nil || !validation.Approved {
t.Fatalf("default item-registry identity validation = %#v, %v; want normalized cross-chunk candidates accepted", validation, validationErr)
}
var candidates struct {
Candidates []struct {
Name string `json:"name"`
} `json:"candidates"`
}
if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Name != "Rope" || candidates.Candidates[1].Name != "Lantern" {
t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err)
}
repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc))
if repeatErr != nil || !reflect.DeepEqual(repeated.Value, result.Value) {
t.Fatalf("repeated normalization = %#v, %v; want stable output", repeated, repeatErr)
} }
} }