Improve D&D registry caching and retire the completed roadmap

This commit is contained in:
2026-08-04 18:29:17 +00:00
parent f5fd115046
commit 29fcad6e9b
7 changed files with 96 additions and 956 deletions

View File

@@ -2,9 +2,7 @@ package registry
import (
"bytes"
"fmt"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -100,26 +98,19 @@ func TestResolveRejectsInvalidReferenceInputs(t *testing.T) {
}
}
func TestResolverValidatesStaticAndOperationReferences(t *testing.T) {
func TestResolverHandlesConstructionAndOperationReferences(t *testing.T) {
placeholder, err := NewResolver(referenceSet())
if err != nil || placeholder.Seeded().Bound() {
t.Fatalf("generated placeholder = %#v, %v; want unbound seed", placeholder, err)
}
validContent := encodeList(t, registryFixture())
invalidSets := []contracts.ReferenceSet{
referenceSet(item([]byte(`{"locations":[`))),
referenceSet(contracts.ReferenceItem{MediaType: "text/plain", Content: validContent}),
referenceSet(item(make([]byte, MaxBytes+1))),
referenceSet(item(validContent), item(validContent)),
malformed := referenceSet(item([]byte(`{"locations":[`)))
if _, err := NewResolver(malformed); err == nil {
t.Fatal("NewResolver(malformed) error = nil")
}
for index, references := range invalidSets {
if _, err := NewResolver(references); err == nil {
t.Fatalf("NewResolver(invalid %d) error = nil", index)
}
if _, err := placeholder.Resolve(references); err == nil {
t.Fatalf("Resolve(invalid %d) error = nil", index)
}
if _, err := placeholder.Resolve(malformed); err == nil {
t.Fatal("Resolve(malformed) error = nil")
}
staticContent := append([]byte(nil), validContent...)
@@ -137,9 +128,12 @@ func TestResolverValidatesStaticAndOperationReferences(t *testing.T) {
if seeded.Seeded().Count() != 2 || seeded.Seeded().CanonicalBytes()[0] != '{' {
t.Fatalf("seeded registry retained construction references: %#v", seeded.Seeded())
}
if fallback, err := seeded.Resolve(contracts.ReferenceSet{}); err != nil || fallback != seeded.Seeded() {
t.Fatalf("fallback = %#v, %v; want seeded registry", fallback, err)
}
}
func TestResolverCachesEquivalentRegistriesConcurrentlyAndIgnoresCallerDigest(t *testing.T) {
func TestResolverReusesEquivalentCanonicalRegistries(t *testing.T) {
resolver, err := NewResolver(referenceSet())
if err != nil {
t.Fatal(err)
@@ -153,57 +147,11 @@ func TestResolverCachesEquivalentRegistriesConcurrentlyAndIgnoresCallerDigest(t
if err != nil {
t.Fatal(err)
}
second, err := resolver.Resolve(firstSet)
if err != nil || first != second {
t.Fatalf("raw cache Resolve() = %p, %p, %v", first, second, err)
}
spaced := append([]byte("\n "), content...)
spaced = append(spaced, '\n')
third, err := resolver.Resolve(referenceSet(contracts.ReferenceItem{MediaType: "APPLICATION/JSON; charset=utf-8", Content: spaced}))
if err != nil || third != first {
t.Fatalf("semantic cache Resolve() = %p, %p, %v", first, third, err)
}
var group sync.WaitGroup
errs := make(chan error, 24)
for range 24 {
group.Add(1)
go func() {
defer group.Done()
resolved, err := resolver.Resolve(firstSet)
if err != nil || resolved != first {
errs <- fmt.Errorf("resolved %p, want %p: %w", resolved, first, err)
}
}()
}
group.Wait()
close(errs)
for err := range errs {
t.Error(err)
}
sharedDigest := "sha256:" + strings.Repeat("0", 64)
firstItem := contracts.ReferenceItem{MediaType: locationcodec.MediaType, Content: content, Digest: sharedDigest}
otherList := registryFixture()
otherList.Locations[0].Name = "Moon Gate"
otherList.Locations[0].ID = identity.DeriveID(otherList.Locations[0].Name, otherList.Locations[0].SourceRefs)
otherContent := encodeList(t, otherList)
otherItem := contracts.ReferenceItem{MediaType: locationcodec.MediaType, Content: otherContent, Digest: sharedDigest}
byDigestFirst, err := resolver.Resolve(referenceSet(firstItem))
if err != nil {
t.Fatal(err)
}
byDigestOther, err := resolver.Resolve(referenceSet(otherItem))
if err != nil || byDigestFirst == byDigestOther || byDigestFirst.Digest() == byDigestOther.Digest() {
t.Fatalf("caller digest aliased different registries: %p / %p, %v", byDigestFirst, byDigestOther, err)
}
firstSet.Slots[ReferenceSlot].Items[0].Content[0] = '['
if got, ok := first.Lookup(registryFixture().Locations[0].ID); !ok || got.Name != "The Tavern" {
t.Fatalf("cached registry retained caller bytes: %#v, %t", got, ok)
}
if fallback, err := resolver.Resolve(contracts.ReferenceSet{}); err != nil || fallback != resolver.Seeded() || fallback.Bound() {
t.Fatalf("fallback = %#v, %v; want unbound seed", fallback, err)
second, err := resolver.Resolve(referenceSet(contracts.ReferenceItem{MediaType: "APPLICATION/JSON; charset=utf-8", Content: spaced}))
if err != nil || second != first {
t.Fatalf("equivalent canonical registry = %p / %p, %v", first, second, err)
}
}

View File

@@ -101,20 +101,8 @@ func TestNormalizerContractsRequiredRegistryAndWarningBounds(t *testing.T) {
if spec.Key != Key || spec.Stage != pipeline.StageNormalize || spec.ExecutionClass != contracts.ExecutionClassDeterministic || spec.ArtifactKind != dnd.LocationOccurrenceListKind {
t.Fatalf("ModuleSpec() = %#v", spec)
}
wantSlots := []contracts.ReferenceSlot{{
Name: LocationRegistryReferenceSlot,
Description: "Required normalized location registry used only for location identity grounding, never as occurrence evidence.",
Required: true,
AcceptedMediaTypes: []string{"application/json"},
AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.LocationListKind},
MaxBytes: LocationRegistryMaxBytes,
}}
if !reflect.DeepEqual(spec.ReferenceSlots, wantSlots) {
t.Fatalf("ModuleSpec().ReferenceSlots = %#v, want %#v", spec.ReferenceSlots, wantSlots)
}
if got := normalizer.ReferenceSlots(); !reflect.DeepEqual(got, wantSlots) {
t.Fatalf("ReferenceSlots() = %#v, want %#v", got, wantSlots)
}
assertLocationRegistryReferenceSlots(t, "ModuleSpec", spec.ReferenceSlots)
assertLocationRegistryReferenceSlots(t, "Normalizer", normalizer.ReferenceSlots())
registry := pipeline.NewNormalizerRegistry()
if err := Register(registry); err != nil {
t.Fatal(err)
@@ -123,9 +111,7 @@ func TestNormalizerContractsRequiredRegistryAndWarningBounds(t *testing.T) {
if !ok {
t.Fatalf("registry missing %q", Key)
}
if !reflect.DeepEqual(registeredSpec.ReferenceSlots, wantSlots) {
t.Fatalf("registered reference slots = %#v, want %#v", registeredSpec.ReferenceSlots, wantSlots)
}
assertLocationRegistryReferenceSlots(t, "registered ModuleSpec", registeredSpec.ReferenceSlots)
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
t.Fatal("DecodeOptions() accepted unknown options")
}
@@ -153,6 +139,26 @@ func TestNormalizerContractsRequiredRegistryAndWarningBounds(t *testing.T) {
}
}
func assertLocationRegistryReferenceSlots(t *testing.T, owner string, slots []contracts.ReferenceSlot) {
t.Helper()
if len(slots) != 1 {
t.Fatalf("%s reference slots = %#v, want one location registry", owner, slots)
}
got := slots[0]
description := got.Description
got.Description = ""
want := contracts.ReferenceSlot{
Name: LocationRegistryReferenceSlot,
Required: true,
AcceptedMediaTypes: []string{"application/json"},
AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.LocationListKind},
MaxBytes: LocationRegistryMaxBytes,
}
if !reflect.DeepEqual(got, want) || strings.TrimSpace(description) == "" {
t.Fatalf("%s location registry slot = %#v, want contract %#v with a nonempty description", owner, slots[0], want)
}
}
func newNormalizer(t *testing.T, references ...contracts.ReferenceSet) *Normalizer {
t.Helper()
normalizer, err := New(Options{}, references...)

View File

@@ -2,10 +2,8 @@ package registry
import (
"bytes"
"fmt"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -114,17 +112,27 @@ func TestResolveRejectsMalformedOrUnsupportedRegistryInput(t *testing.T) {
func TestResolverReusesEquivalentCanonicalRegistries(t *testing.T) {
set := listReferenceSet(t, registryFixture())
resolver, err := NewResolver(set)
resolver, err := NewResolver(contracts.ReferenceSet{})
if err != nil {
t.Fatal(err)
}
resolved, err := resolver.Resolve(set)
if err != nil || resolved != resolver.Seeded() {
t.Fatalf("Resolve() = %p, %v; seeded %p", resolved, err, resolver.Seeded())
first, err := resolver.Resolve(set)
if err != nil {
t.Fatal(err)
}
content := set.Slots[ReferenceSlot].Items[0].Content
equivalent := append([]byte("\n "), content...)
equivalent = append(equivalent, '\n')
second, err := resolver.Resolve(referenceSet(contracts.ReferenceItem{
MediaType: "APPLICATION/JSON; charset=utf-8",
Content: equivalent,
}))
if err != nil || second != first {
t.Fatalf("equivalent canonical registry = %p / %p, %v", first, second, err)
}
}
func TestResolverValidatesStaticAndOperationReferences(t *testing.T) {
func TestResolverHandlesConstructionAndOperationReferences(t *testing.T) {
placeholder, err := NewResolver(referenceSet())
if err != nil || placeholder.Seeded().Bound() {
t.Fatalf("generated placeholder = %#v, %v; want unbound seed", placeholder, err)
@@ -132,22 +140,12 @@ func TestResolverValidatesStaticAndOperationReferences(t *testing.T) {
valid := listReferenceSet(t, registryFixture())
validContent := valid.Slots[ReferenceSlot].Items[0].Content
invalidSets := []contracts.ReferenceSet{
referenceSet(contracts.ReferenceItem{MediaType: npccodec.MediaType, Content: []byte(`{"npcs":[`)}),
referenceSet(contracts.ReferenceItem{MediaType: "text/plain", Content: validContent}),
referenceSet(contracts.ReferenceItem{MediaType: npccodec.MediaType, Content: make([]byte, MaxBytes+1)}),
referenceSet(
contracts.ReferenceItem{MediaType: npccodec.MediaType, Content: validContent},
contracts.ReferenceItem{MediaType: npccodec.MediaType, Content: validContent},
),
malformed := referenceSet(contracts.ReferenceItem{MediaType: npccodec.MediaType, Content: []byte(`{"npcs":[`)})
if _, err := NewResolver(malformed); err == nil {
t.Fatal("NewResolver(malformed) error = nil")
}
for index, references := range invalidSets {
if _, err := NewResolver(references); err == nil {
t.Fatalf("NewResolver(invalid %d) error = nil", index)
}
if _, err := placeholder.Resolve(references); err == nil {
t.Fatalf("Resolve(invalid %d) error = nil", index)
}
if _, err := placeholder.Resolve(malformed); err == nil {
t.Fatal("Resolve(malformed) error = nil")
}
staticContent := append([]byte(nil), validContent...)
@@ -161,71 +159,8 @@ func TestResolverValidatesStaticAndOperationReferences(t *testing.T) {
if seeded.Seeded().Count() != 2 || seeded.Seeded().CanonicalBytes()[0] != '{' {
t.Fatalf("seeded registry retained construction references: %#v", seeded.Seeded())
}
}
func TestResolverCachesEquivalentRegistriesConcurrentlyAndIgnoresCallerDigest(t *testing.T) {
resolver, err := NewResolver(referenceSet())
if err != nil {
t.Fatal(err)
}
content := listReferenceSet(t, registryFixture()).Slots[ReferenceSlot].Items[0].Content
references := referenceSet(contracts.ReferenceItem{MediaType: npccodec.MediaType, Content: content})
first, err := resolver.Resolve(references)
if err != nil {
t.Fatal(err)
}
second, err := resolver.Resolve(references)
if err != nil || second != first {
t.Fatalf("raw reuse = %p / %p, %v", first, second, err)
}
spaced := append([]byte("\n "), content...)
spaced = append(spaced, '\n')
equivalent, err := resolver.Resolve(referenceSet(contracts.ReferenceItem{MediaType: "APPLICATION/JSON; charset=utf-8", Content: spaced}))
if err != nil || equivalent != first {
t.Fatalf("semantic reuse = %p / %p, %v", first, equivalent, err)
}
const callers = 24
var group sync.WaitGroup
errors := make(chan error, callers)
for range callers {
group.Add(1)
go func() {
defer group.Done()
resolved, err := resolver.Resolve(references)
if err != nil || resolved != first {
errors <- fmt.Errorf("resolved %p, want %p: %w", resolved, first, err)
}
}()
}
group.Wait()
close(errors)
for err := range errors {
t.Error(err)
}
sharedDigest := "sha256:" + strings.Repeat("0", 64)
firstItem := contracts.ReferenceItem{MediaType: npccodec.MediaType, Content: content, Digest: sharedDigest}
otherList := registryFixture()
otherList.NPCs[0].Name = "The Greencloak"
otherList.NPCs[0].ID = identity.DeriveID(otherList.NPCs[0].Name)
otherContent := listReferenceSet(t, otherList).Slots[ReferenceSlot].Items[0].Content
otherItem := contracts.ReferenceItem{MediaType: npccodec.MediaType, Content: otherContent, Digest: sharedDigest}
byDigestFirst, err := resolver.Resolve(referenceSet(firstItem))
if err != nil {
t.Fatal(err)
}
byDigestOther, err := resolver.Resolve(referenceSet(otherItem))
if err != nil || byDigestFirst == byDigestOther || byDigestFirst.Digest() == byDigestOther.Digest() {
t.Fatalf("caller digest aliased different registries: %p / %p, %v", byDigestFirst, byDigestOther, err)
}
content[0] = '['
if npc, ok := first.Lookup("Mira Thorn"); !ok || npc.Name != "Mira Thorn" {
t.Fatalf("resolved registry retained operation bytes: %#v, %t", npc, ok)
}
if fallback, err := resolver.Resolve(contracts.ReferenceSet{}); err != nil || fallback != resolver.Seeded() || fallback.Bound() {
t.Fatalf("fallback = %#v, %v; want unbound seed", fallback, err)
if fallback, err := seeded.Resolve(contracts.ReferenceSet{}); err != nil || fallback != seeded.Seeded() {
t.Fatalf("fallback = %#v, %v; want seeded registry", fallback, err)
}
}

View File

@@ -426,7 +426,7 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
if len(occurrenceExtractSpec.ReferenceSlots) != 5 || len(occurrenceNormalizeSpec.ReferenceSlots) != 1 {
t.Fatalf("location occurrence reference slots = %#v / %#v, want extractor campaign context and normalizer registry only", occurrenceExtractSpec.ReferenceSlots, occurrenceNormalizeSpec.ReferenceSlots)
}
if !locationRegistrySlot.Required || !reflect.DeepEqual(locationRegistrySlot.AcceptedMediaTypes, []string{"application/json"}) || !reflect.DeepEqual(locationRegistrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.LocationListKind}) || locationRegistrySlot.MaxBytes != 1048576 || !reflect.DeepEqual(locationRegistrySlot, occurrenceNormalizeRegistrySlot) {
if !locationRegistrySlot.Required || !reflect.DeepEqual(locationRegistrySlot.AcceptedMediaTypes, []string{"application/json"}) || !reflect.DeepEqual(locationRegistrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.LocationListKind}) || locationRegistrySlot.MaxBytes != 1048576 || !sameReferenceSlotContract(locationRegistrySlot, occurrenceNormalizeRegistrySlot) {
t.Fatalf("location registry slots disagree: %#v / %#v", occurrenceExtractSpec.ReferenceSlots, occurrenceNormalizeSpec.ReferenceSlots)
}
for _, name := range []string{"party", "roster", "players", "glossary"} {
@@ -445,7 +445,7 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
}
extractRegistrySlot := referenceSlot(interactionExtractSpec.ReferenceSlots, "npcs")
normalizeRegistrySlot := referenceSlot(interactionNormalizeSpec.ReferenceSlots, "npcs")
if !extractRegistrySlot.Required || !reflect.DeepEqual(extractRegistrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCListKind}) || !reflect.DeepEqual(extractRegistrySlot, normalizeRegistrySlot) {
if !extractRegistrySlot.Required || !reflect.DeepEqual(extractRegistrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCListKind}) || !sameReferenceSlotContract(extractRegistrySlot, normalizeRegistrySlot) {
t.Fatalf("NPC interaction registry slots disagree: %#v / %#v", interactionExtractSpec.ReferenceSlots, interactionNormalizeSpec.ReferenceSlots)
}
}
@@ -508,6 +508,12 @@ func referenceSlot(slots []contracts.ReferenceSlot, name string) contracts.Refer
return contracts.ReferenceSlot{}
}
func sameReferenceSlotContract(first, second contracts.ReferenceSlot) bool {
first.Description = ""
second.Description = ""
return reflect.DeepEqual(first, second)
}
func TestAppendNPCListsPreservesOrderAndArrayPresence(t *testing.T) {
tests := []struct {
name string

View File

@@ -28,6 +28,13 @@ type Item struct {
Content []byte
}
// validatedItem borrows its content from the supplied reference set. Callers
// must copy content before passing it to a callback that may retain it.
type validatedItem struct {
mediaType string
content []byte
}
// Config supplies the domain-owned operations needed to prepare immutable
// registry views. Absent and Load must return values whose mutable state is not
// exposed to callers. Load receives owned bytes and may retain them. Errors
@@ -111,7 +118,7 @@ func (r *Resolver[V]) Resolve(references contracts.ReferenceSet) (V, error) {
return r.seeded.value, nil
}
item, _, err := resolveOptionalSingleItem(references, r.config.Reference, r.mediaType)
item, _, err := validateOptionalSingleItem(references, r.config.Reference, r.mediaType)
if err != nil {
var zero V
return zero, err
@@ -124,7 +131,7 @@ func (r *Resolver[V]) Resolve(references contracts.ReferenceSet) (V, error) {
return cached.value, nil
}
resolved, err := r.load(item.Content)
resolved, err := r.load(append([]byte(nil), item.content...))
if err != nil {
var zero V
return zero, err
@@ -147,23 +154,30 @@ func (r *Resolver[V]) Resolve(references contracts.ReferenceSet) (V, error) {
// ResolveOptionalSingleItem validates and copies one optional registry item.
// A missing slot returns present=false. A present slot must contain exactly one
// item, even when it represents an operation-time generated reference.
func ResolveOptionalSingleItem(references contracts.ReferenceSet, spec ReferenceSpec) (item Item, present bool, err error) {
func ResolveOptionalSingleItem(references contracts.ReferenceSet, spec ReferenceSpec) (Item, bool, error) {
normalized, mediaType, err := normalizeReferenceSpec(spec)
if err != nil {
return Item{}, false, err
}
return resolveOptionalSingleItem(references, normalized, mediaType)
item, present, err := validateOptionalSingleItem(references, normalized, mediaType)
if err != nil || !present {
return Item{}, present, err
}
return Item{
MediaType: item.mediaType,
Content: append([]byte(nil), item.content...),
}, true, nil
}
func (r *Resolver[V]) resolveUncached(references contracts.ReferenceSet) (preparedView[V], error) {
item, present, err := resolveOptionalSingleItem(references, r.config.Reference, r.mediaType)
item, present, err := validateOptionalSingleItem(references, r.config.Reference, r.mediaType)
if err != nil {
return preparedView[V]{}, err
}
if !present {
return r.absent()
}
return r.load(item.Content)
return r.load(append([]byte(nil), item.content...))
}
func (r *Resolver[V]) absent() (preparedView[V], error) {
@@ -221,32 +235,32 @@ func normalizeReferenceSpec(spec ReferenceSpec) (ReferenceSpec, string, error) {
return spec, mediaType, nil
}
func resolveOptionalSingleItem(references contracts.ReferenceSet, spec ReferenceSpec, acceptedMediaType string) (Item, bool, error) {
func validateOptionalSingleItem(references contracts.ReferenceSet, spec ReferenceSpec, acceptedMediaType string) (validatedItem, bool, error) {
slot, present := references.Slots[spec.SlotName]
if !present {
return Item{}, false, nil
return validatedItem{}, false, nil
}
if len(slot.Items) != 1 {
return Item{}, true, fmt.Errorf("reference slot %q must contain exactly one item", spec.SlotName)
return validatedItem{}, true, fmt.Errorf("reference slot %q must contain exactly one item", spec.SlotName)
}
item := slot.Items[0]
mediaType, _, err := mime.ParseMediaType(item.MediaType)
if err != nil {
return Item{}, true, fmt.Errorf("reference slot %q item media type is invalid", spec.SlotName)
return validatedItem{}, true, fmt.Errorf("reference slot %q item media type is invalid", spec.SlotName)
}
mediaType = strings.ToLower(mediaType)
if !strings.EqualFold(mediaType, acceptedMediaType) {
return Item{}, true, fmt.Errorf("reference slot %q item media type must be %s", spec.SlotName, acceptedMediaType)
return validatedItem{}, true, fmt.Errorf("reference slot %q item media type must be %s", spec.SlotName, acceptedMediaType)
}
if int64(len(item.Content)) > spec.MaxBytes {
return Item{}, true, fmt.Errorf("reference slot %q item is %d bytes, limit %d", spec.SlotName, len(item.Content), spec.MaxBytes)
return validatedItem{}, true, fmt.Errorf("reference slot %q item is %d bytes, limit %d", spec.SlotName, len(item.Content), spec.MaxBytes)
}
return Item{MediaType: mediaType, Content: append([]byte(nil), item.Content...)}, true, nil
return validatedItem{mediaType: mediaType, content: item.Content}, true, nil
}
func rawReferenceKey(item Item) string {
sum := sha256.Sum256(item.Content)
return item.MediaType + "\x00sha256:" + hex.EncodeToString(sum[:])
func rawReferenceKey(item validatedItem) string {
sum := sha256.Sum256(item.content)
return item.mediaType + "\x00sha256:" + hex.EncodeToString(sum[:])
}
func sameIdentity[V any](first, second preparedView[V]) bool {