// Package itemoccurrences normalizes merged D&D item-occurrence candidates. package itemoccurrences import ( "context" "fmt" "sort" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" itemoccurrencemodel "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/itemoccurrences" itemregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/registry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" ) const ( Key = "dnd/item-occurrences" normalizationPolicy = "dnd.item_occurrences.normalize.v1" NormalizationPolicy = normalizationPolicy ReasonCodeNameCanonicalized = "item_occurrence_name_canonicalized" ReasonCodeUnknownItemID = "item_occurrence_unknown_item_id" ReasonCodeSourceRefsNormalized = "source_references_normalized" ReasonCodeOccurrencesReordered = "item_occurrences_reordered" ReasonCodeDuplicateCollapsed = "duplicate_item_occurrence_collapsed" ReasonCodeWarningsOmitted = "item_occurrence_normalization_warnings_omitted" ) const ( ItemRegistryReferenceSlot = itemregistry.ReferenceSlot ItemRegistryMaxBytes = itemregistry.MaxBytes ) var requiredCapabilities = []string{"merged"} var providedCapabilities = []string{"normalized"} var _ contracts.Normalizer[dnd.ItemOccurrenceList] = (*Normalizer)(nil) var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil) var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil) type Options struct{} type Normalizer struct{ itemResolver *itemregistry.Resolver } func New(_ Options, references ...contracts.ReferenceSet) (*Normalizer, error) { if len(references) > 1 { return nil, normalizerErrorf("at most one reference set may be supplied") } var referenceSet contracts.ReferenceSet if len(references) == 1 { referenceSet = references[0] } resolver, err := itemregistry.NewResolver(referenceSet) if err != nil { return nil, normalizerErrorf("prepare item registry: %w", err) } return &Normalizer{itemResolver: resolver}, nil } func (n *Normalizer) Key() string { return Key } func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } func (n *Normalizer) ManifestMetadata() map[string]any { if n == nil || n.itemResolver == nil { return nil } metadata := map[string]any{"normalization_policy": normalizationPolicy, "comparison_policy": shared.TextComparisonPolicy} seeded := n.itemResolver.Seeded() if seeded.Bound() { metadata["item_registry_digest"] = seeded.Digest() metadata["item_count"] = seeded.Count() } return metadata } func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { if n == nil || n.itemResolver == nil { return nil } return []pipeline.CheckpointFingerprint{ {Name: "normalization_policy", Value: normalizationPolicy}, {Name: "comparison_policy", Value: shared.TextComparisonPolicy}, {Name: "item_registry", Value: n.itemResolver.Seeded().ProjectionDigest()}, } } func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.ItemOccurrenceList]) (contracts.TypedNormalizeResult[dnd.ItemOccurrenceList], error) { if n == nil || n.itemResolver == nil { return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("normalizer must not be nil") } if ctx == nil { return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("context must not be nil") } if err := ctx.Err(); err != nil { return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("context error before normalize: %w", err) } registry, err := n.itemResolver.Resolve(req.References) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("resolve item registry: %w", err) } if !registry.Bound() { return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{}, normalizerErrorf("item registry reference is required") } index := source.NewDocumentIndex(req.Source) value, warnings := normalizeList(req.MergeOutput.Value, index, shared.NewSourceRefOrderFromIndex(index), registry) return contracts.TypedNormalizeResult[dnd.ItemOccurrenceList]{Value: value, Warnings: warnings}, nil } type normalizedRecord struct { occurrence dnd.ItemOccurrence identity string inputIndex int } func normalizeList(input dnd.ItemOccurrenceList, index source.DocumentIndex, order shared.SourceRefOrder, registry *itemregistry.Registry) (dnd.ItemOccurrenceList, []contracts.Warning) { if input.Occurrences == nil { return dnd.ItemOccurrenceList{}, nil } records := make([]normalizedRecord, len(input.Occurrences)) warnings := make([]contracts.Warning, 0) for index, inputOccurrence := range input.Occurrences { occurrence, changedFields, found, refsChanged := normalizeOccurrence(inputOccurrence, order, registry) records[index] = normalizedRecord{occurrence: occurrence, identity: itemoccurrencemodel.CanonicalExactIdentity(occurrence), inputIndex: index} if len(changedFields) != 0 { warnings = append(warnings, contracts.Warning{ Scope: occurrenceScope(index), ReasonCode: ReasonCodeNameCanonicalized, Message: fmt.Sprintf("input index %d: normalized display whitespace in %s", index, diagnostics.Aggregate("fields", changedFields)), }) } if found && inputOccurrence.Name != occurrence.Name { warnings = append(warnings, contracts.Warning{Scope: occurrenceScope(index), ReasonCode: ReasonCodeNameCanonicalized, Message: fmt.Sprintf("input index %d: item name canonicalized from %s to %s", index, diagnostics.Quote(inputOccurrence.Name), diagnostics.Quote(occurrence.Name))}) } if !found { warnings = append(warnings, contracts.Warning{Scope: occurrenceScope(index), ReasonCode: ReasonCodeUnknownItemID, Message: fmt.Sprintf("input index %d: item ID %s is not in the supplied registry", index, diagnostics.Quote(inputOccurrence.ItemID))}) } if refsChanged { warnings = append(warnings, contracts.Warning{ Scope: occurrenceScope(index), ReasonCode: ReasonCodeSourceRefsNormalized, Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)", index, len(inputOccurrence.SourceRefs), len(occurrence.SourceRefs)), }) } } sort.SliceStable(records, func(left, right int) bool { return itemoccurrencemodel.CanonicalLess(order, records[left].occurrence, records[right].occurrence) }) for position, record := range records { if position == record.inputIndex { continue } warnings = append(warnings, contracts.Warning{ Scope: occurrenceScope(record.inputIndex), ReasonCode: ReasonCodeOccurrencesReordered, Message: fmt.Sprintf("input index %d moved to normalized position %d", record.inputIndex, position), }) } output, duplicateWarnings := collapseDuplicates(records, index) warnings = append(warnings, duplicateWarnings...) return dnd.ItemOccurrenceList{Occurrences: output}, diagnostics.LimitWarnings(warnings, "item_occurrences", ReasonCodeWarningsOmitted) } func normalizeOccurrence(input dnd.ItemOccurrence, order shared.SourceRefOrder, registry *itemregistry.Registry) (dnd.ItemOccurrence, []string, bool, bool) { output := cloneOccurrence(input) canonical, found := registry.LookupID(input.ItemID) if found { output.Name = canonical.Name } changedFields := make([]string, 0, 2) for _, field := range []struct { name string value *string }{ {name: "from", value: &output.From}, {name: "to", value: &output.To}, } { trimmed := itemoccurrencemodel.DisplayValue(*field.value) if *field.value != trimmed { *field.value = trimmed changedFields = append(changedFields, field.name) } } output.SourceRefs = order.Canonicalize(input.SourceRefs) return output, changedFields, found, !itemoccurrencemodel.SourceRefsEqual(input.SourceRefs, output.SourceRefs) } func cloneOccurrence(input dnd.ItemOccurrence) dnd.ItemOccurrence { output := input if input.Quantity != nil { quantity := *input.Quantity output.Quantity = &quantity } if input.SourceRefs != nil { output.SourceRefs = append([]source.SourceRef(nil), input.SourceRefs...) } return output } type duplicateGroup struct { retainedIndex int retainedRecord int removed []int } func collapseDuplicates(records []normalizedRecord, index source.DocumentIndex) ([]dnd.ItemOccurrence, []contracts.Warning) { if len(records) == 0 { return make([]dnd.ItemOccurrence, 0), nil } keep := make([]bool, len(records)) groups := make([]duplicateGroup, 0) groupByKey := make(map[string][]int) for recordIndex, record := range records { if !itemoccurrencemodel.ValidSourceRefs(index, record.occurrence.SourceRefs) { keep[recordIndex] = true continue } groupIndex := -1 for _, candidate := range groupByKey[record.identity] { if itemoccurrencemodel.CanonicalExactEqual(records[groups[candidate].retainedRecord].occurrence, record.occurrence) { groupIndex = candidate break } } if groupIndex < 0 { groupIndex = len(groups) groupByKey[record.identity] = append(groupByKey[record.identity], groupIndex) groups = append(groups, duplicateGroup{retainedIndex: record.inputIndex, retainedRecord: recordIndex}) keep[recordIndex] = true continue } groups[groupIndex].removed = append(groups[groupIndex].removed, record.inputIndex) } output := make([]dnd.ItemOccurrence, 0, len(records)) for recordIndex, record := range records { if keep[recordIndex] { output = append(output, cloneOccurrence(record.occurrence)) } } warnings := make([]contracts.Warning, 0) for _, group := range groups { if len(group.removed) != 0 { warnings = append(warnings, duplicateWarning(group.retainedIndex, group.removed)) } } return output, warnings } func duplicateWarning(retainedIndex int, removed []int) contracts.Warning { issues := make([]string, len(removed)) for index, removedIndex := range removed { issues[index] = fmt.Sprintf("removed input index %d", removedIndex) } return contracts.Warning{ Scope: occurrenceScope(retainedIndex), ReasonCode: ReasonCodeDuplicateCollapsed, Message: diagnostics.Aggregate( fmt.Sprintf("duplicate item occurrence collapsed; retained input index %d", retainedIndex), issues), } } func occurrenceScope(index int) string { return fmt.Sprintf("occurrences[%d]", index) } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.ItemOccurrenceListKind, ReferenceSlots: referenceSlots(), } } func Register(registry *pipeline.NormalizerRegistry) error { return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.ItemOccurrenceList], error) { options, err := DecodeOptions(request.Options) if err != nil { return nil, err } return New(options, request.References) }) } func referenceSlots() []contracts.ReferenceSlot { return []contracts.ReferenceSlot{{ Name: ItemRegistryReferenceSlot, Description: "Required normalized item registry used only for item identity grounding.", Required: true, AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.ItemRegistryKind}, MaxBytes: ItemRegistryMaxBytes, }} } func DecodeOptions(options map[string]any) (Options, error) { if err := pipeline.RejectUnknownOptions(options); err != nil { return Options{}, normalizerErrorf("%w", err) } return Options{}, nil } func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } func normalizerErrorf(format string, args ...any) error { return fmt.Errorf("dnd item occurrences normalizer: "+format, args...) }