Move item occurrences to canonical namespace
This commit is contained in:
312
internal/modules/dnd/normalize/itemoccurrences/normalizer.go
Normal file
312
internal/modules/dnd/normalize/itemoccurrences/normalizer.go
Normal file
@@ -0,0 +1,312 @@
|
||||
// Package itemoccurrences normalizes merged D&D item-occurrence candidates.
|
||||
package itemoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
itemoccurrencemodel "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/itemoccurrences"
|
||||
itemregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/registry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/item-occurrences"
|
||||
normalizationPolicy = "dnd.item_occurrences.normalize.v1"
|
||||
NormalizationPolicy = normalizationPolicy
|
||||
|
||||
ReasonCodeNameCanonicalized = "item_occurrence_name_canonicalized"
|
||||
ReasonCodeUnknownItemID = "item_occurrence_unknown_item_id"
|
||||
ReasonCodeSourceRefsNormalized = "source_references_normalized"
|
||||
ReasonCodeOccurrencesReordered = "item_occurrences_reordered"
|
||||
ReasonCodeDuplicateCollapsed = "duplicate_item_occurrence_collapsed"
|
||||
ReasonCodeWarningsOmitted = "item_occurrence_normalization_warnings_omitted"
|
||||
)
|
||||
|
||||
const (
|
||||
ItemRegistryReferenceSlot = itemregistry.ReferenceSlot
|
||||
ItemRegistryMaxBytes = itemregistry.MaxBytes
|
||||
)
|
||||
|
||||
var requiredCapabilities = []string{"merged"}
|
||||
var providedCapabilities = []string{"normalized"}
|
||||
|
||||
var _ contracts.Normalizer[dnd.ItemOccurrenceList] = (*Normalizer)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Normalizer struct{ itemResolver *itemregistry.Resolver }
|
||||
|
||||
func New(_ Options, references ...contracts.ReferenceSet) (*Normalizer, error) {
|
||||
if len(references) > 1 {
|
||||
return nil, normalizerErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
var referenceSet contracts.ReferenceSet
|
||||
if len(references) == 1 {
|
||||
referenceSet = references[0]
|
||||
}
|
||||
resolver, err := itemregistry.NewResolver(referenceSet)
|
||||
if err != nil {
|
||||
return nil, normalizerErrorf("prepare item registry: %w", err)
|
||||
}
|
||||
return &Normalizer{itemResolver: resolver}, nil
|
||||
}
|
||||
|
||||
func (n *Normalizer) Key() string { return Key }
|
||||
|
||||
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (n *Normalizer) ManifestMetadata() map[string]any {
|
||||
if n == nil || n.itemResolver == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := map[string]any{"normalization_policy": normalizationPolicy}
|
||||
seeded := n.itemResolver.Seeded()
|
||||
if seeded.Bound() {
|
||||
metadata["item_registry_digest"] = seeded.Digest()
|
||||
metadata["item_count"] = seeded.Count()
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if n == nil || n.itemResolver == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "normalization_policy", Value: normalizationPolicy},
|
||||
{Name: "item_registry", Value: n.itemResolver.Seeded().ProjectionDigest()},
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.ItemOccurrenceList]) (contracts.TypedNormalizeResult[dnd.ItemOccurrenceList], error) {
|
||||
if n == nil || n.itemResolver == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("normalizer must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("context error before normalize: %w", err)
|
||||
}
|
||||
|
||||
registry, err := n.itemResolver.Resolve(req.References)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("resolve item registry: %w", err)
|
||||
}
|
||||
if !registry.Bound() {
|
||||
return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("item registry reference is required")
|
||||
}
|
||||
index := source.NewDocumentIndex(req.Source)
|
||||
value, warnings := normalizeList(req.MergeOutput.Value, index, shared.NewSourceRefOrderFromIndex(index), registry)
|
||||
return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{Value: value, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
type normalizedRecord struct {
|
||||
occurrence dnd.ItemOccurrence
|
||||
inputIndex int
|
||||
}
|
||||
|
||||
func normalizeList(input dnd.ItemOccurrenceList, index source.DocumentIndex, order shared.SourceRefOrder, registry *itemregistry.Registry) (dnd.ItemOccurrenceList, []contracts.Warning) {
|
||||
if input.Occurrences == nil {
|
||||
return dnd.ItemOccurrenceList{}, nil
|
||||
}
|
||||
|
||||
records := make([]normalizedRecord, len(input.Occurrences))
|
||||
warnings := make([]contracts.Warning, 0)
|
||||
for index, inputOccurrence := range input.Occurrences {
|
||||
occurrence, changedFields, found, refsChanged := normalizeOccurrence(inputOccurrence, order, registry)
|
||||
records[index] = normalizedRecord{occurrence: occurrence, inputIndex: index}
|
||||
if len(changedFields) != 0 {
|
||||
warnings = append(warnings, contracts.Warning{
|
||||
Scope: occurrenceScope(index),
|
||||
ReasonCode: ReasonCodeNameCanonicalized,
|
||||
Message: fmt.Sprintf("input index %d: normalized display whitespace in %s", index,
|
||||
diagnostics.Aggregate("fields", changedFields)),
|
||||
})
|
||||
}
|
||||
if found && inputOccurrence.Name != occurrence.Name {
|
||||
warnings = append(warnings, contracts.Warning{Scope: occurrenceScope(index), ReasonCode: ReasonCodeNameCanonicalized,
|
||||
Message: fmt.Sprintf("input index %d: item name canonicalized from %s to %s", index, diagnostics.Quote(inputOccurrence.Name), diagnostics.Quote(occurrence.Name))})
|
||||
}
|
||||
if !found {
|
||||
warnings = append(warnings, contracts.Warning{Scope: occurrenceScope(index), ReasonCode: ReasonCodeUnknownItemID,
|
||||
Message: fmt.Sprintf("input index %d: item ID %s is not in the supplied registry", index, diagnostics.Quote(inputOccurrence.ItemID))})
|
||||
}
|
||||
if refsChanged {
|
||||
warnings = append(warnings, contracts.Warning{
|
||||
Scope: occurrenceScope(index),
|
||||
ReasonCode: ReasonCodeSourceRefsNormalized,
|
||||
Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)",
|
||||
index, len(inputOccurrence.SourceRefs), len(occurrence.SourceRefs)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.SliceStable(records, func(left, right int) bool {
|
||||
return itemoccurrencemodel.Less(order, records[left].occurrence, records[right].occurrence)
|
||||
})
|
||||
for position, record := range records {
|
||||
if position == record.inputIndex {
|
||||
continue
|
||||
}
|
||||
warnings = append(warnings, contracts.Warning{
|
||||
Scope: occurrenceScope(record.inputIndex),
|
||||
ReasonCode: ReasonCodeOccurrencesReordered,
|
||||
Message: fmt.Sprintf("input index %d moved to normalized position %d", record.inputIndex, position),
|
||||
})
|
||||
}
|
||||
|
||||
output, duplicateWarnings := collapseDuplicates(records, index, order)
|
||||
warnings = append(warnings, duplicateWarnings...)
|
||||
return dnd.ItemOccurrenceList{Occurrences: output}, diagnostics.LimitWarnings(warnings, "item_occurrences", ReasonCodeWarningsOmitted)
|
||||
}
|
||||
|
||||
func normalizeOccurrence(input dnd.ItemOccurrence, order shared.SourceRefOrder, registry *itemregistry.Registry) (dnd.ItemOccurrence, []string, bool, bool) {
|
||||
output := cloneOccurrence(input)
|
||||
canonical, found := registry.LookupID(input.ItemID)
|
||||
if found {
|
||||
output.Name = canonical.Name
|
||||
}
|
||||
changedFields := make([]string, 0, 2)
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
value *string
|
||||
}{
|
||||
{name: "from", value: &output.From},
|
||||
{name: "to", value: &output.To},
|
||||
} {
|
||||
trimmed := itemoccurrencemodel.DisplayValue(*field.value)
|
||||
if *field.value != trimmed {
|
||||
*field.value = trimmed
|
||||
changedFields = append(changedFields, field.name)
|
||||
}
|
||||
}
|
||||
output.SourceRefs = order.Canonicalize(input.SourceRefs)
|
||||
return output, changedFields, found, !itemoccurrencemodel.SourceRefsEqual(input.SourceRefs, output.SourceRefs)
|
||||
}
|
||||
|
||||
func cloneOccurrence(input dnd.ItemOccurrence) dnd.ItemOccurrence {
|
||||
output := input
|
||||
if input.Quantity != nil {
|
||||
quantity := *input.Quantity
|
||||
output.Quantity = &quantity
|
||||
}
|
||||
if input.SourceRefs != nil {
|
||||
output.SourceRefs = append([]source.SourceRef(nil), input.SourceRefs...)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
type duplicateGroup struct {
|
||||
retainedIndex int
|
||||
removed []int
|
||||
}
|
||||
|
||||
func collapseDuplicates(records []normalizedRecord, index source.DocumentIndex, order shared.SourceRefOrder) ([]dnd.ItemOccurrence, []contracts.Warning) {
|
||||
if len(records) == 0 {
|
||||
return make([]dnd.ItemOccurrence, 0), nil
|
||||
}
|
||||
|
||||
keep := make([]bool, len(records))
|
||||
groups := make([]duplicateGroup, 0)
|
||||
groupByKey := make(map[string]int)
|
||||
for recordIndex, record := range records {
|
||||
if !itemoccurrencemodel.ValidSourceRefs(index, record.occurrence.SourceRefs) {
|
||||
keep[recordIndex] = true
|
||||
continue
|
||||
}
|
||||
key := itemoccurrencemodel.ExactIdentity(order, record.occurrence)
|
||||
groupIndex, exists := groupByKey[key]
|
||||
if !exists {
|
||||
groupByKey[key] = len(groups)
|
||||
groups = append(groups, duplicateGroup{retainedIndex: record.inputIndex})
|
||||
keep[recordIndex] = true
|
||||
continue
|
||||
}
|
||||
groups[groupIndex].removed = append(groups[groupIndex].removed, record.inputIndex)
|
||||
}
|
||||
|
||||
output := make([]dnd.ItemOccurrence, 0, len(records))
|
||||
for recordIndex, record := range records {
|
||||
if keep[recordIndex] {
|
||||
output = append(output, cloneOccurrence(record.occurrence))
|
||||
}
|
||||
}
|
||||
warnings := make([]contracts.Warning, 0)
|
||||
for _, group := range groups {
|
||||
if len(group.removed) != 0 {
|
||||
warnings = append(warnings, duplicateWarning(group.retainedIndex, group.removed))
|
||||
}
|
||||
}
|
||||
return output, warnings
|
||||
}
|
||||
|
||||
func duplicateWarning(retainedIndex int, removed []int) contracts.Warning {
|
||||
issues := make([]string, len(removed))
|
||||
for index, removedIndex := range removed {
|
||||
issues[index] = fmt.Sprintf("removed input index %d", removedIndex)
|
||||
}
|
||||
return contracts.Warning{
|
||||
Scope: occurrenceScope(retainedIndex),
|
||||
ReasonCode: ReasonCodeDuplicateCollapsed,
|
||||
Message: diagnostics.Aggregate(
|
||||
fmt.Sprintf("duplicate item occurrence collapsed; retained input index %d", retainedIndex), issues),
|
||||
}
|
||||
}
|
||||
|
||||
func occurrenceScope(index int) string { return fmt.Sprintf("occurrences[%d]", index) }
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageNormalize,
|
||||
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.ItemOccurrenceListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.ItemOccurrenceList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
func referenceSlots() []contracts.ReferenceSlot {
|
||||
return []contracts.ReferenceSlot{{
|
||||
Name: ItemRegistryReferenceSlot,
|
||||
Description: "Required normalized item registry used only for item identity grounding.",
|
||||
Required: true,
|
||||
AcceptedMediaTypes: []string{"application/json"},
|
||||
AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.ItemRegistryKind},
|
||||
MaxBytes: ItemRegistryMaxBytes,
|
||||
}}
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, normalizerErrorf("%w", err)
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
func normalizerErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("dnd item occurrences normalizer: "+format, args...)
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package itemoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
|
||||
itemidentity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
)
|
||||
|
||||
func registryReferences(t *testing.T) contracts.ReferenceSet {
|
||||
t.Helper()
|
||||
content, err := itemcodec.New().Encode(dnd.ItemRegistry{Items: []dnd.Item{{ID: itemidentity.DeriveID("Torch"), Name: "Torch", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ItemRegistryReferenceSlot: {Slot: contracts.ReferenceSlot{Name: ItemRegistryReferenceSlot}, Items: []contracts.ReferenceItem{{SlotName: ItemRegistryReferenceSlot, Content: content, MediaType: "application/json", ArtifactKind: dnd.ItemRegistryKind}}}}}
|
||||
}
|
||||
|
||||
func TestNormalizeCanonicalizesRegistryNameAndRetainsUnknownValues(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1}, {ID: 2}}}
|
||||
refs := registryReferences(t)
|
||||
normalizer, err := New(Options{}, refs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := itemidentity.DeriveID("Torch")
|
||||
input := dnd.ItemOccurrenceList{Occurrences: []dnd.ItemOccurrence{
|
||||
{ItemID: id, Name: "torch", Kind: dnd.ItemOccurrenceKindDiscovered, SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 2, EndUnitID: 2}}},
|
||||
{ItemID: "unknown", Name: "Unknown", Kind: dnd.ItemOccurrenceKindDiscovered, SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}},
|
||||
}}
|
||||
result, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[dnd.ItemOccurrenceList]{Source: doc, References: refs, MergeOutput: contracts.MergeArtifact[dnd.ItemOccurrenceList]{Value: input}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Value.Occurrences[1].Name != "Torch" || result.Value.Occurrences[0].Name != "Unknown" {
|
||||
t.Fatalf("occurrences = %#v", result.Value.Occurrences)
|
||||
}
|
||||
if !hasWarning(result.Warnings, ReasonCodeNameCanonicalized) || !hasWarning(result.Warnings, ReasonCodeUnknownItemID) {
|
||||
t.Fatalf("warnings = %#v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRequiresRegistryAndRegistersSlot(t *testing.T) {
|
||||
normalizer, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[dnd.ItemOccurrenceList]{MergeOutput: contracts.MergeArtifact[dnd.ItemOccurrenceList]{Value: dnd.ItemOccurrenceList{Occurrences: []dnd.ItemOccurrence{}}}})
|
||||
if err == nil {
|
||||
t.Fatal("Normalize() error = nil")
|
||||
}
|
||||
if len(ModuleSpec().ReferenceSlots) != 1 || !ModuleSpec().ReferenceSlots[0].Required {
|
||||
t.Fatalf("ModuleSpec() = %#v", ModuleSpec())
|
||||
}
|
||||
registry := pipeline.NewNormalizerRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func hasWarning(warnings []contracts.Warning, reason string) bool {
|
||||
for _, warning := range warnings {
|
||||
if warning.ReasonCode == reason {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user