Add D&D location occurrence normalizer
This commit is contained in:
371
internal/modules/dnd/normalize/locationoccurrences/normalizer.go
Normal file
371
internal/modules/dnd/normalize/locationoccurrences/normalizer.go
Normal file
@@ -0,0 +1,371 @@
|
||||
// Package locationoccurrences normalizes merged D&D location occurrence candidates.
|
||||
package locationoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
locationregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/registry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/location-occurrences"
|
||||
normalizationPolicy = "dnd.location_occurrences.normalize.v1"
|
||||
NormalizationPolicy = normalizationPolicy
|
||||
|
||||
ReasonCodeNameCanonicalized = "location_occurrence_name_canonicalized"
|
||||
ReasonCodeUnknownLocationID = "location_occurrence_unknown_location_id"
|
||||
ReasonCodeSourceRefsNormalized = "source_references_normalized"
|
||||
ReasonCodeOccurrencesReordered = "location_occurrences_reordered"
|
||||
ReasonCodeDuplicateCollapsed = "duplicate_location_occurrence_collapsed"
|
||||
ReasonCodeWarningsOmitted = "location_occurrence_normalization_warnings_omitted"
|
||||
)
|
||||
|
||||
const (
|
||||
LocationRegistryReferenceSlot = locationregistry.ReferenceSlot
|
||||
LocationRegistryMaxBytes = locationregistry.MaxBytes
|
||||
)
|
||||
|
||||
var requiredCapabilities = []string{"merged"}
|
||||
var providedCapabilities = []string{"normalized"}
|
||||
|
||||
var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
||||
Glossary: "Optional campaign glossary reference material used only for location-occurrence disambiguation.",
|
||||
Party: "Optional party roster reference material used only for location-occurrence disambiguation.",
|
||||
Players: "Optional player list reference material used only for location-occurrence disambiguation.",
|
||||
Roster: "Deprecated alias for party roster reference material used only for location-occurrence disambiguation.",
|
||||
}
|
||||
|
||||
var _ contracts.Normalizer[dnd.LocationOccurrenceList] = (*Normalizer)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Normalizer struct {
|
||||
locationResolver *locationregistry.Resolver
|
||||
}
|
||||
|
||||
func New(_ Options, references ...contracts.ReferenceSet) (*Normalizer, error) {
|
||||
if len(references) > 1 {
|
||||
return nil, normalizerErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
var referenceSet contracts.ReferenceSet
|
||||
if len(references) == 1 {
|
||||
referenceSet = references[0]
|
||||
}
|
||||
resolver, err := locationregistry.NewResolver(referenceSet)
|
||||
if err != nil {
|
||||
return nil, normalizerErrorf("prepare location registry: %w", err)
|
||||
}
|
||||
return &Normalizer{locationResolver: resolver}, nil
|
||||
}
|
||||
|
||||
func (n *Normalizer) Key() string { return Key }
|
||||
|
||||
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (n *Normalizer) ManifestMetadata() map[string]any {
|
||||
if n == nil || n.locationResolver == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := map[string]any{"normalization_policy": normalizationPolicy}
|
||||
seeded := n.locationResolver.Seeded()
|
||||
if seeded.Bound() {
|
||||
metadata["location_registry_digest"] = seeded.Digest()
|
||||
metadata["location_count"] = seeded.Count()
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if n == nil || n.locationResolver == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "normalization_policy", Value: normalizationPolicy},
|
||||
{Name: "location_registry", Value: n.locationResolver.Seeded().ProjectionDigest()},
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.LocationOccurrenceList]) (contracts.TypedNormalizeResult[dnd.LocationOccurrenceList], error) {
|
||||
if n == nil || n.locationResolver == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationOccurrenceList]{}, normalizerErrorf("normalizer must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationOccurrenceList]{}, normalizerErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationOccurrenceList]{}, normalizerErrorf("context error before normalize: %w", err)
|
||||
}
|
||||
registry, err := n.locationResolver.Resolve(req.References)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationOccurrenceList]{}, normalizerErrorf("resolve location registry: %w", err)
|
||||
}
|
||||
if !registry.Bound() {
|
||||
return contracts.TypedNormalizeResult[dnd.LocationOccurrenceList]{}, normalizerErrorf("location registry reference is required")
|
||||
}
|
||||
index := source.NewDocumentIndex(req.Source)
|
||||
value, warnings := normalizeList(req.MergeOutput.Value, index, shared.NewSourceRefOrderFromIndex(index), registry)
|
||||
return contracts.TypedNormalizeResult[dnd.LocationOccurrenceList]{Value: value, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
type normalizedRecord struct {
|
||||
occurrence dnd.LocationOccurrence
|
||||
inputIndex int
|
||||
}
|
||||
|
||||
type nameCanonicalization struct{ from, to string }
|
||||
|
||||
func normalizeList(input dnd.LocationOccurrenceList, documentIndex source.DocumentIndex, order shared.SourceRefOrder, registry *locationregistry.Registry) (dnd.LocationOccurrenceList, []contracts.Warning) {
|
||||
if input.Occurrences == nil {
|
||||
return dnd.LocationOccurrenceList{}, nil
|
||||
}
|
||||
records := make([]normalizedRecord, len(input.Occurrences))
|
||||
warnings := make([]contracts.Warning, 0)
|
||||
for index, inputOccurrence := range input.Occurrences {
|
||||
occurrence, change, found, refsChanged := normalizeOccurrence(inputOccurrence, order, registry)
|
||||
records[index] = normalizedRecord{occurrence: occurrence, inputIndex: index}
|
||||
if change != nil {
|
||||
warnings = append(warnings, contracts.Warning{Scope: occurrenceScope(index), ReasonCode: ReasonCodeNameCanonicalized,
|
||||
Message: fmt.Sprintf("input index %d: location name canonicalized from %s to %s", index, diagnostics.Quote(change.from), diagnostics.Quote(change.to))})
|
||||
}
|
||||
if !found {
|
||||
warnings = append(warnings, contracts.Warning{Scope: occurrenceScope(index), ReasonCode: ReasonCodeUnknownLocationID,
|
||||
Message: fmt.Sprintf("input index %d: location ID %s is not in the supplied registry", index, diagnostics.Quote(inputOccurrence.LocationID))})
|
||||
}
|
||||
if refsChanged {
|
||||
warnings = append(warnings, contracts.Warning{Scope: occurrenceScope(index), ReasonCode: ReasonCodeSourceRefsNormalized,
|
||||
Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)", index, len(inputOccurrence.SourceRefs), len(occurrence.SourceRefs))})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(records, func(left, right int) bool {
|
||||
return lessOccurrence(order, records[left].occurrence, records[right].occurrence)
|
||||
})
|
||||
for position, record := range records {
|
||||
if position != record.inputIndex {
|
||||
warnings = append(warnings, contracts.Warning{Scope: occurrenceScope(record.inputIndex), ReasonCode: ReasonCodeOccurrencesReordered,
|
||||
Message: fmt.Sprintf("input index %d moved to normalized position %d by canonical occurrence order", record.inputIndex, position)})
|
||||
}
|
||||
}
|
||||
output, duplicateWarnings := collapseDuplicates(records, documentIndex)
|
||||
warnings = append(warnings, duplicateWarnings...)
|
||||
return dnd.LocationOccurrenceList{Occurrences: output}, diagnostics.LimitWarnings(warnings, "location_occurrences", ReasonCodeWarningsOmitted)
|
||||
}
|
||||
|
||||
func normalizeOccurrence(input dnd.LocationOccurrence, order shared.SourceRefOrder, registry *locationregistry.Registry) (dnd.LocationOccurrence, *nameCanonicalization, bool, bool) {
|
||||
output := cloneOccurrence(input)
|
||||
canonical, found := registry.Lookup(input.LocationID)
|
||||
if found {
|
||||
output.Name = canonical.Name
|
||||
}
|
||||
var change *nameCanonicalization
|
||||
if input.Name != output.Name {
|
||||
change = &nameCanonicalization{from: input.Name, to: output.Name}
|
||||
}
|
||||
output.SourceRefs = order.Canonicalize(input.SourceRefs)
|
||||
return output, change, found, !sourceRefsEqual(input.SourceRefs, output.SourceRefs)
|
||||
}
|
||||
|
||||
func cloneOccurrence(input dnd.LocationOccurrence) dnd.LocationOccurrence {
|
||||
input.SourceRefs = append([]source.SourceRef(nil), input.SourceRefs...)
|
||||
return input
|
||||
}
|
||||
|
||||
func sourceRefsEqual(left, right []source.SourceRef) bool {
|
||||
if (left == nil) != (right == nil) || len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
for index := range left {
|
||||
if left[index] != right[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type duplicateGroup struct {
|
||||
retainedIndex int
|
||||
removed []int
|
||||
}
|
||||
|
||||
func collapseDuplicates(records []normalizedRecord, documentIndex source.DocumentIndex) ([]dnd.LocationOccurrence, []contracts.Warning) {
|
||||
if len(records) == 0 {
|
||||
return make([]dnd.LocationOccurrence, 0), nil
|
||||
}
|
||||
keep := make([]bool, len(records))
|
||||
groups := make([]duplicateGroup, 0)
|
||||
groupByKey := make(map[string]int)
|
||||
for index, record := range records {
|
||||
if !validSourceRefs(documentIndex, record.occurrence.SourceRefs) {
|
||||
keep[index] = true
|
||||
continue
|
||||
}
|
||||
key := exactIdentity(record.occurrence)
|
||||
groupIndex, exists := groupByKey[key]
|
||||
if !exists {
|
||||
groupByKey[key] = len(groups)
|
||||
groups = append(groups, duplicateGroup{retainedIndex: record.inputIndex})
|
||||
keep[index] = true
|
||||
continue
|
||||
}
|
||||
groups[groupIndex].removed = append(groups[groupIndex].removed, record.inputIndex)
|
||||
}
|
||||
output := make([]dnd.LocationOccurrence, 0, len(records))
|
||||
for index, record := range records {
|
||||
if keep[index] {
|
||||
output = append(output, cloneOccurrence(record.occurrence))
|
||||
}
|
||||
}
|
||||
warnings := make([]contracts.Warning, 0)
|
||||
for _, group := range groups {
|
||||
if len(group.removed) > 0 {
|
||||
warnings = append(warnings, duplicateWarning(group.retainedIndex, group.removed))
|
||||
}
|
||||
}
|
||||
return output, warnings
|
||||
}
|
||||
|
||||
func validSourceRefs(index source.DocumentIndex, refs []source.SourceRef) bool {
|
||||
if len(refs) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, ref := range refs {
|
||||
if index.ValidateRef(ref) != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func exactIdentity(occurrence dnd.LocationOccurrence) string {
|
||||
var key strings.Builder
|
||||
writeKeyString(&key, occurrence.LocationID)
|
||||
writeKeyString(&key, occurrence.Name)
|
||||
writeKeyString(&key, string(occurrence.Kind))
|
||||
for _, ref := range occurrence.SourceRefs {
|
||||
writeKeyString(&key, ref.SourceID)
|
||||
writeKeyInt(&key, ref.StartUnitID)
|
||||
writeKeyInt(&key, ref.EndUnitID)
|
||||
}
|
||||
return key.String()
|
||||
}
|
||||
|
||||
func writeKeyString(builder *strings.Builder, value string) {
|
||||
builder.WriteString(strconv.Itoa(len(value)))
|
||||
builder.WriteByte(':')
|
||||
builder.WriteString(value)
|
||||
}
|
||||
|
||||
func writeKeyInt(builder *strings.Builder, value int) {
|
||||
builder.WriteString(strconv.Itoa(value))
|
||||
builder.WriteByte(';')
|
||||
}
|
||||
|
||||
func lessOccurrence(order shared.SourceRefOrder, left, right dnd.LocationOccurrence) bool {
|
||||
leftPosition, leftHasEvidence := order.EarliestValid(left.SourceRefs)
|
||||
rightPosition, rightHasEvidence := order.EarliestValid(right.SourceRefs)
|
||||
if leftHasEvidence != rightHasEvidence {
|
||||
return leftHasEvidence
|
||||
}
|
||||
if leftHasEvidence && leftPosition != rightPosition {
|
||||
return leftPosition < rightPosition
|
||||
}
|
||||
if left.LocationID != right.LocationID {
|
||||
return left.LocationID < right.LocationID
|
||||
}
|
||||
if left.Name != right.Name {
|
||||
return left.Name < right.Name
|
||||
}
|
||||
if kindOrder(left.Kind) != kindOrder(right.Kind) {
|
||||
return kindOrder(left.Kind) < kindOrder(right.Kind)
|
||||
}
|
||||
if left.Kind != right.Kind {
|
||||
return left.Kind < right.Kind
|
||||
}
|
||||
return sourceRefsLess(order, left.SourceRefs, right.SourceRefs)
|
||||
}
|
||||
|
||||
func kindOrder(kind dnd.LocationOccurrenceKind) int {
|
||||
switch kind {
|
||||
case dnd.LocationOccurrenceKindVisited:
|
||||
return 0
|
||||
case dnd.LocationOccurrenceKindPlanned:
|
||||
return 1
|
||||
case dnd.LocationOccurrenceKindRecalled:
|
||||
return 2
|
||||
case dnd.LocationOccurrenceKindMentioned:
|
||||
return 3
|
||||
default:
|
||||
return 4
|
||||
}
|
||||
}
|
||||
|
||||
func sourceRefsLess(order shared.SourceRefOrder, left, right []source.SourceRef) bool {
|
||||
for index := 0; index < len(left) && index < len(right); index++ {
|
||||
if left[index] == right[index] {
|
||||
continue
|
||||
}
|
||||
return order.Less(left[index], right[index])
|
||||
}
|
||||
return len(left) < len(right)
|
||||
}
|
||||
|
||||
func duplicateWarning(retainedIndex int, removed []int) contracts.Warning {
|
||||
issues := make([]string, len(removed))
|
||||
for index, removedIndex := range removed {
|
||||
issues[index] = fmt.Sprintf("removed input index %d", removedIndex)
|
||||
}
|
||||
return contracts.Warning{Scope: occurrenceScope(retainedIndex), ReasonCode: ReasonCodeDuplicateCollapsed,
|
||||
Message: diagnostics.Aggregate(fmt.Sprintf("duplicate location occurrence collapsed; retained input index %d", retainedIndex), issues)}
|
||||
}
|
||||
|
||||
func occurrenceScope(index int) string { return fmt.Sprintf("occurrences[%d]", index) }
|
||||
|
||||
func referenceSlots() []contracts.ReferenceSlot {
|
||||
slots := shared.ReferenceSlots(referenceSlotDescriptions)
|
||||
slots = append(slots, contracts.ReferenceSlot{
|
||||
Name: LocationRegistryReferenceSlot, Description: "Required normalized location registry used only for occurrence identity grounding, never as occurrence evidence.",
|
||||
Required: true, AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.LocationListKind}, MaxBytes: LocationRegistryMaxBytes,
|
||||
})
|
||||
sort.Slice(slots, func(left, right int) bool { return slots[left].Name < slots[right].Name })
|
||||
return slots
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.LocationOccurrenceListKind, ReferenceSlots: referenceSlots()}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.NormalizerRegistry) error {
|
||||
return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.LocationOccurrenceList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, normalizerErrorf("%w", err)
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
func normalizerErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("dnd location occurrences normalizer: "+format, args...)
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package locationoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
locationcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locations"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
)
|
||||
|
||||
func TestNormalizeCanonicalizesNamesByIDAndClonesInputs(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 30}, {ID: 10}}}
|
||||
locations := registryLocations("The Tavern", "The Tavern")
|
||||
normalizer := newNormalizer(t, registryReferences(t, locations))
|
||||
input := dnd.LocationOccurrenceList{Occurrences: []dnd.LocationOccurrence{{
|
||||
LocationID: locations.Locations[1].ID, Name: " a tavern ", Kind: dnd.LocationOccurrenceKindVisited,
|
||||
SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}, {SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}, {SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}},
|
||||
}}}
|
||||
before := cloneList(input)
|
||||
result, err := normalizer.Normalize(context.Background(), normalizeRequest(input, doc, contracts.ReferenceSet{}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
occurrence := result.Value.Occurrences[0]
|
||||
if occurrence.Name != locations.Locations[1].Name || !reflect.DeepEqual(occurrence.SourceRefs, []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}, {SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}) {
|
||||
t.Fatalf("normalized occurrence = %#v", occurrence)
|
||||
}
|
||||
if !hasWarning(result.Warnings, ReasonCodeNameCanonicalized) || !hasWarning(result.Warnings, ReasonCodeSourceRefsNormalized) || !reflect.DeepEqual(input, before) {
|
||||
t.Fatalf("warnings/input = %#v/%#v", result.Warnings, input)
|
||||
}
|
||||
second, err := normalizer.Normalize(context.Background(), normalizeRequest(result.Value, doc, contracts.ReferenceSet{}))
|
||||
if err != nil || !reflect.DeepEqual(second.Value, result.Value) || len(second.Warnings) != 0 {
|
||||
t.Fatalf("second normalization = %#v, %v", second, err)
|
||||
}
|
||||
result.Value.Occurrences[0].SourceRefs[0].StartUnitID = 999
|
||||
if input.Occurrences[0].SourceRefs[0].StartUnitID == 999 {
|
||||
t.Fatal("normalized source references share input storage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeKeepsSameNamedIDsAndDistinctEvidence(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 50}, {ID: 10}, {ID: 90}}}
|
||||
locations := registryLocations("The Tavern", "The Tavern")
|
||||
first, second := locations.Locations[0], locations.Locations[1]
|
||||
ref := func(unit int) source.SourceRef {
|
||||
return source.SourceRef{SourceID: doc.ID, StartUnitID: unit, EndUnitID: unit}
|
||||
}
|
||||
firstMention := dnd.LocationOccurrence{LocationID: first.ID, Name: first.Name, Kind: dnd.LocationOccurrenceKindMentioned, SourceRefs: []source.SourceRef{ref(50)}}
|
||||
input := dnd.LocationOccurrenceList{Occurrences: []dnd.LocationOccurrence{
|
||||
{LocationID: second.ID, Name: second.Name, Kind: dnd.LocationOccurrenceKindMentioned, SourceRefs: []source.SourceRef{ref(90)}},
|
||||
firstMention,
|
||||
firstMention,
|
||||
{LocationID: first.ID, Name: first.Name, Kind: dnd.LocationOccurrenceKindVisited, SourceRefs: []source.SourceRef{ref(50)}},
|
||||
{LocationID: first.ID, Name: first.Name, Kind: dnd.LocationOccurrenceKindPlanned, SourceRefs: []source.SourceRef{ref(50)}},
|
||||
{LocationID: first.ID, Name: first.Name, Kind: dnd.LocationOccurrenceKindRecalled, SourceRefs: []source.SourceRef{ref(50)}},
|
||||
{LocationID: first.ID, Name: first.Name, Kind: dnd.LocationOccurrenceKindMentioned, SourceRefs: []source.SourceRef{ref(10)}},
|
||||
{LocationID: first.ID, Name: first.Name, Kind: dnd.LocationOccurrenceKindMentioned, SourceRefs: []source.SourceRef{ref(999)}},
|
||||
}}
|
||||
result, err := newNormalizer(t, registryReferences(t, locations)).Normalize(context.Background(), normalizeRequest(input, doc, contracts.ReferenceSet{}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := result.Value.Occurrences
|
||||
if len(got) != 7 || got[0].Kind != dnd.LocationOccurrenceKindVisited || got[1].Kind != dnd.LocationOccurrenceKindPlanned || got[2].Kind != dnd.LocationOccurrenceKindRecalled || got[3].Kind != dnd.LocationOccurrenceKindMentioned || got[3].SourceRefs[0].StartUnitID != 50 || got[4].SourceRefs[0].StartUnitID != 10 || got[5].LocationID != second.ID || got[6].SourceRefs[0].StartUnitID != 999 {
|
||||
t.Fatalf("canonical occurrences = %#v", got)
|
||||
}
|
||||
if !hasWarning(result.Warnings, ReasonCodeDuplicateCollapsed) || !hasWarning(result.Warnings, ReasonCodeOccurrencesReordered) {
|
||||
t.Fatalf("warnings = %#v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePreservesUnknownIDsAndMalformedOperationRegistry(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 10}}}
|
||||
locations := registryLocations("The Mill")
|
||||
unknown := dnd.LocationOccurrence{LocationID: "location:sha256:unknown", Name: "The Mill", Kind: "unexpected", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}
|
||||
result, err := newNormalizer(t, registryReferences(t, locations)).Normalize(context.Background(), normalizeRequest(dnd.LocationOccurrenceList{Occurrences: []dnd.LocationOccurrence{unknown}}, doc, contracts.ReferenceSet{}))
|
||||
if err != nil || !reflect.DeepEqual(result.Value.Occurrences[0], unknown) || !hasWarning(result.Warnings, ReasonCodeUnknownLocationID) {
|
||||
t.Fatalf("unknown normalization = %#v, %v", result, err)
|
||||
}
|
||||
malformed := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{LocationRegistryReferenceSlot: {
|
||||
Items: []contracts.ReferenceItem{{SlotName: LocationRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"private":"registry evidence"}`)}},
|
||||
}}}
|
||||
if _, err := newNormalizer(t).Normalize(context.Background(), normalizeRequest(dnd.LocationOccurrenceList{}, doc, malformed)); err == nil || !strings.Contains(err.Error(), "resolve location registry") || strings.Contains(err.Error(), "registry evidence") {
|
||||
t.Fatalf("operation registry error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizerContractsRequiredRegistryAndWarningBounds(t *testing.T) {
|
||||
if _, err := New(Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
normalizer := newNormalizer(t, registryReferences(t, registryLocations("The Mill")))
|
||||
if spec := ModuleSpec(); spec.Key != Key || spec.Stage != pipeline.StageNormalize || spec.ExecutionClass != contracts.ExecutionClassDeterministic || spec.ArtifactKind != dnd.LocationOccurrenceListKind {
|
||||
t.Fatalf("ModuleSpec() = %#v", spec)
|
||||
}
|
||||
var locationSlot contracts.ReferenceSlot
|
||||
for _, slot := range ModuleSpec().ReferenceSlots {
|
||||
if slot.Name == LocationRegistryReferenceSlot {
|
||||
locationSlot = slot
|
||||
}
|
||||
}
|
||||
if !locationSlot.Required || !reflect.DeepEqual(locationSlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.LocationListKind}) || locationSlot.MaxBytes != LocationRegistryMaxBytes {
|
||||
t.Fatalf("location slot = %#v", locationSlot)
|
||||
}
|
||||
registry := pipeline.NewNormalizerRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := registry.Spec(Key); !ok {
|
||||
t.Fatalf("registry missing %q", Key)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown options")
|
||||
}
|
||||
if metadata := normalizer.ManifestMetadata(); metadata["normalization_policy"] != normalizationPolicy || metadata["location_registry_digest"] == "" || metadata["location_count"] != 1 {
|
||||
t.Fatalf("metadata = %#v", metadata)
|
||||
}
|
||||
if fingerprints := normalizer.CheckpointFingerprints(); len(fingerprints) != 2 || fingerprints[1].Name != "location_registry" || fingerprints[1].Value == "" {
|
||||
t.Fatalf("fingerprints = %#v", fingerprints)
|
||||
}
|
||||
if _, err := newNormalizer(t).Normalize(context.Background(), normalizeRequest(dnd.LocationOccurrenceList{}, nil, contracts.ReferenceSet{})); err == nil || !strings.Contains(err.Error(), "required") {
|
||||
t.Fatalf("unbound registry error = %v", err)
|
||||
}
|
||||
|
||||
count := diagnostics.MaxWarnings + 5
|
||||
doc := &source.SourceDocument{ID: "session", Units: make([]source.SourceUnit, count)}
|
||||
input := dnd.LocationOccurrenceList{Occurrences: make([]dnd.LocationOccurrence, count)}
|
||||
location := registryLocations("The Mill").Locations[0]
|
||||
for index := range doc.Units {
|
||||
doc.Units[index].ID = index + 1
|
||||
input.Occurrences[index] = dnd.LocationOccurrence{LocationID: location.ID, Name: "not canonical", Kind: dnd.LocationOccurrenceKindMentioned, SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: count - index, EndUnitID: count - index}}}
|
||||
}
|
||||
bounded, err := newNormalizer(t, registryReferences(t, dnd.LocationList{Locations: []dnd.Location{location}})).Normalize(context.Background(), normalizeRequest(input, doc, contracts.ReferenceSet{}))
|
||||
if err != nil || len(bounded.Warnings) != diagnostics.MaxWarnings || bounded.Warnings[len(bounded.Warnings)-1].ReasonCode != ReasonCodeWarningsOmitted {
|
||||
t.Fatalf("bounded warnings = %#v, %v", bounded.Warnings, err)
|
||||
}
|
||||
}
|
||||
|
||||
func newNormalizer(t *testing.T, references ...contracts.ReferenceSet) *Normalizer {
|
||||
t.Helper()
|
||||
normalizer, err := New(Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
return normalizer
|
||||
}
|
||||
|
||||
func normalizeRequest(value dnd.LocationOccurrenceList, doc *source.SourceDocument, references contracts.ReferenceSet) contracts.TypedNormalizeRequest[dnd.LocationOccurrenceList] {
|
||||
return contracts.TypedNormalizeRequest[dnd.LocationOccurrenceList]{Source: doc, MergeOutput: contracts.MergeArtifact[dnd.LocationOccurrenceList]{Value: value}, References: references}
|
||||
}
|
||||
|
||||
func registryLocations(names ...string) dnd.LocationList {
|
||||
locations := make([]dnd.Location, len(names))
|
||||
for index, name := range names {
|
||||
refs := []source.SourceRef{{SourceID: "registry", StartUnitID: index + 1, EndUnitID: index + 1}}
|
||||
locations[index] = dnd.Location{ID: identity.DeriveID(name, refs), Name: name, SourceRefs: refs}
|
||||
}
|
||||
return dnd.LocationList{Locations: locations}
|
||||
}
|
||||
|
||||
func registryReferences(t *testing.T, locations dnd.LocationList) contracts.ReferenceSet {
|
||||
t.Helper()
|
||||
content, err := locationcodec.New().Encode(locations)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{LocationRegistryReferenceSlot: {
|
||||
Items: []contracts.ReferenceItem{{SlotName: LocationRegistryReferenceSlot, MediaType: locationcodec.MediaType, Content: content}},
|
||||
}}}
|
||||
}
|
||||
|
||||
func cloneList(input dnd.LocationOccurrenceList) dnd.LocationOccurrenceList {
|
||||
output := dnd.LocationOccurrenceList{Occurrences: make([]dnd.LocationOccurrence, len(input.Occurrences))}
|
||||
for index, occurrence := range input.Occurrences {
|
||||
output.Occurrences[index] = occurrence
|
||||
output.Occurrences[index].SourceRefs = append([]source.SourceRef(nil), occurrence.SourceRefs...)
|
||||
}
|
||||
if input.Occurrences == nil {
|
||||
output.Occurrences = nil
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func hasWarning(warnings []contracts.Warning, code string) bool {
|
||||
for _, warning := range warnings {
|
||||
if warning.ReasonCode == code {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user