Index collection policy retirement

docs/roadmap/implementation.md

@@ -860,6 +860,16 @@ must replace Q scans of all P entries. No exact syscall/timing test
 or public complexity guarantee is needed.
 Focused check: `go test ./internal/engine -run 'Collection|Retirement|Sequence|Inheritance'`.
 
+Status: **complete** (E01). Prior completed commits: `2736ddf`, `deb9aa5`,
+`ae3408a`, `ea27f17`, `08a8c25`, `877c497`, `8aba43d`, `7bf2f0c`, `0f9ecf1`,
+`119b42c`, `247a12b`, `4fcff5f`, `12d350a`, `08f2a8f`, `826062a`, `e4a5ecb`,
+`3122e2f`, `6f6130f`, `4fd3bee`, `4bf940c`, `d8bf6ab`, `f185821`, `a8ae2e9`,
+`8832449`, and `549fca2`. Validation passed:
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'Collection|Retirement|Sequence|Inheritance' -count=1`,
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./... -count=1`,
+and `git diff --check`.
+
 ## Stage 27 — Borrow Lower Policy And Produce Sparse Strict-Tier State
 
 Finding: first part of **E02**; coordinates **E03**.

internal/engine/collection_policy_index.go

@@ -0,0 +1,137 @@
+package engine
+
+// collectionPolicyRetirementIndex locates only assembly policies whose YAML
+// identity or record-owned state can be cleared by an ancestor replacement.
+// Mapping keys and sequence indexes remain distinct through pathSegment keys.
+type collectionPolicyRetirementIndex struct {
+	children map[pathSegment]*collectionPolicyRetirementIndex
+	entryKey string
+	hasEntry bool
+}
+
+func newCollectionPolicyRetirementIndex(scope collectionPolicyScope) *collectionPolicyRetirementIndex {
+	index := &collectionPolicyRetirementIndex{}
+	for _, entry := range scope.collections {
+		if collectionPolicyHasRetirementState(entry.policy) {
+			index.set(entry)
+		}
+	}
+	return index
+}
+
+func (index *collectionPolicyRetirementIndex) set(entry collectionPolicyEntry) {
+	node := index
+	for _, segment := range entry.destination.segments {
+		if node.children == nil {
+			node.children = make(map[pathSegment]*collectionPolicyRetirementIndex)
+		}
+		if node.children[segment] == nil {
+			node.children[segment] = &collectionPolicyRetirementIndex{}
+		}
+		node = node.children[segment]
+	}
+	node.entryKey = entry.destination.String()
+	node.hasEntry = true
+}
+
+func (index *collectionPolicyRetirementIndex) remove(destination Path) {
+	if index == nil {
+		return
+	}
+	nodes := make([]*collectionPolicyRetirementIndex, 1, len(destination.segments)+1)
+	nodes[0] = index
+	node := index
+	for _, segment := range destination.segments {
+		node = node.children[segment]
+		if node == nil {
+			return
+		}
+		nodes = append(nodes, node)
+	}
+	node.entryKey = ""
+	node.hasEntry = false
+	pruneCollectionPolicyIndex(nodes, destination.segments)
+}
+
+func (index *collectionPolicyRetirementIndex) retire(scope *collectionPolicyScope, ancestor Path) {
+	if index == nil || scope == nil {
+		return
+	}
+	nodes := make([]*collectionPolicyRetirementIndex, 1, len(ancestor.segments)+1)
+	nodes[0] = index
+	node := index
+	for _, segment := range ancestor.segments {
+		node = node.children[segment]
+		if node == nil {
+			return
+		}
+		nodes = append(nodes, node)
+	}
+	index.clearSubtree(scope, node)
+	pruneCollectionPolicyIndex(nodes, ancestor.segments)
+}
+
+func (index *collectionPolicyRetirementIndex) clearSubtree(scope *collectionPolicyScope, node *collectionPolicyRetirementIndex) {
+	if node.hasEntry {
+		if entry, found := scope.collections[node.entryKey]; found {
+			clearCollectionIdentity(&entry.policy)
+			scope.collections[node.entryKey] = entry
+		}
+		node.entryKey = ""
+		node.hasEntry = false
+	}
+	for segment, child := range node.children {
+		index.clearSubtree(scope, child)
+		if child.empty() {
+			delete(node.children, segment)
+		}
+	}
+	if len(node.children) == 0 {
+		node.children = nil
+	}
+}
+
+func (index *collectionPolicyRetirementIndex) empty() bool {
+	return index == nil || !index.hasEntry && len(index.children) == 0
+}
+
+func pruneCollectionPolicyIndex(nodes []*collectionPolicyRetirementIndex, segments []pathSegment) {
+	for position := len(segments) - 1; position >= 0; position-- {
+		child := nodes[position+1]
+		if !child.empty() {
+			return
+		}
+		delete(nodes[position].children, segments[position])
+	}
+}
+
+func collectionPolicyHasRetirementState(policy collectionPolicy) bool {
+	return policy.identityFields != nil || policy.identityPresent || policy.identityLocation != nil ||
+		policy.recordOrder != nil || policy.recordPolicies != nil
+}
+
+func (assembly *assemblyAccumulator) setCollectionPolicy(destination Path, policy collectionPolicy) {
+	if assembly == nil || assembly.current == nil {
+		return
+	}
+	if assembly.collectionPolicyRetirement == nil {
+		assembly.collectionPolicyRetirement = newCollectionPolicyRetirementIndex(assembly.current.collectionPolicies)
+	}
+	assembly.current.collectionPolicies.set(destination, policy)
+	entry := assembly.current.collectionPolicies.collections[destination.String()]
+	if collectionPolicyHasRetirementState(entry.policy) {
+		assembly.collectionPolicyRetirement.set(entry)
+	} else {
+		assembly.collectionPolicyRetirement.remove(destination)
+	}
+}
+
+func (assembly *assemblyAccumulator) retireCollectionPoliciesAtOrBelow(ancestor Path) {
+	if assembly == nil || assembly.current == nil {
+		return
+	}
+	if assembly.collectionPolicyRetirement == nil {
+		assembly.collectionPolicyRetirement = newCollectionPolicyRetirementIndex(assembly.current.collectionPolicies)
+	}
+	assembly.collectionPolicyRetirement.retire(&assembly.current.collectionPolicies, ancestor)
+}

internal/engine/collection_retirement_test.go

@@ -2,7 +2,10 @@ package engine
 
 import (
 	"errors"
+	"fmt"
 	"testing"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
 )
 
 func TestSourceReplacementRetiresIdentityForEveryValueKind(t *testing.T) {
@@ -109,3 +112,132 @@ func TestReplacementRetirementPreservesApplicationConstraintsAndRetainedKeys(t *
 		}
 	}
 }
+
+func TestAssemblyPolicyRetirementUsesSemanticPathSegments(t *testing.T) {
+	t.Parallel()
+	var policies collectionPolicyScope
+	slashPath := Keys("branch/name", "items")
+	separateKeysPath := Keys("branch", "name", "items")
+	numericKeyPath := Keys("items", "0", "named")
+	sequenceIndexPath := Keys("items").Index(0).Key("indexed")
+	policies.set(slashPath, collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
+	policies.set(separateKeysPath, collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
+	policies.set(numericKeyPath, collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
+	policies.set(sequenceIndexPath, collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
+
+	assembly := newAssemblyAccumulator(&composedDocument{collectionPolicies: policies})
+	assembly.retireCollectionPoliciesAtOrBelow(Keys("branch/name"))
+	assertRetirement := func(path Path, identityPresent bool) {
+		t.Helper()
+		policy, found := assembly.current.collectionPolicies.at(path)
+		if !found || policy.identityPresent != identityPresent {
+			t.Fatalf("policy at %s = %#v (found %t), want identity present %t", path, policy, found, identityPresent)
+		}
+	}
+	assertRetirement(slashPath, false)
+	assertRetirement(separateKeysPath, true)
+
+	assembly.retireCollectionPoliciesAtOrBelow(Keys("items", "0"))
+	assertRetirement(numericKeyPath, false)
+	assertRetirement(sequenceIndexPath, true)
+	assembly.retireCollectionPoliciesAtOrBelow(Keys("items").Index(0))
+	assertRetirement(sequenceIndexPath, false)
+}
+
+func TestAssemblyPolicyRetirementPrunesAndReindexesAroundModeOnlyEntries(t *testing.T) {
+	t.Parallel()
+	identityPath := Keys("catalog", "items")
+	modePath := Keys("catalog", "settings")
+	var policies collectionPolicyScope
+	policies.set(identityPath, collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
+	policies.set(modePath, collectionPolicy{explicitMode: sequenceModeAppend, explicitModePresent: true})
+	assembly := newAssemblyAccumulator(&composedDocument{collectionPolicies: policies})
+
+	assembly.retireCollectionPoliciesAtOrBelow(Keys("catalog"))
+	retired, found := assembly.current.collectionPolicies.at(identityPath)
+	if !found || retired.identityPresent || !assembly.collectionPolicyRetirement.empty() {
+		t.Fatalf("first retirement left identity or stale index: %#v, index %#v", retired, assembly.collectionPolicyRetirement)
+	}
+	mode, found := assembly.current.collectionPolicies.at(modePath)
+	if !found || mode.explicitMode != sequenceModeAppend || !mode.explicitModePresent {
+		t.Fatalf("mode-only policy changed during retirement: %#v, found %t", mode, found)
+	}
+
+	assembly.retireCollectionPoliciesAtOrBelow(Keys("catalog"))
+	newPath := Keys("catalog", "items", "nested")
+	assembly.setCollectionPolicy(newPath, collectionPolicy{identityFields: []string{"name"}, identityPresent: true})
+	if assembly.collectionPolicyRetirement.empty() {
+		t.Fatal("new identity-bearing policy was not reindexed")
+	}
+	assembly.retireCollectionPoliciesAtOrBelow(Keys("catalog"))
+	newPolicy, found := assembly.current.collectionPolicies.at(newPath)
+	if !found || newPolicy.identityPresent || !assembly.collectionPolicyRetirement.empty() {
+		t.Fatalf("repeated ancestor retirement left new identity or stale index: %#v, index %#v", newPolicy, assembly.collectionPolicyRetirement)
+	}
+	mode, found = assembly.current.collectionPolicies.at(modePath)
+	if !found || mode.explicitMode != sequenceModeAppend || !mode.explicitModePresent {
+		t.Fatalf("repeated retirement changed retained mode: %#v, found %t", mode, found)
+	}
+}
+
+func TestDisjointScalarReplacementsRetainIndexedCollectionPolicies(t *testing.T) {
+	t.Parallel()
+	const policyCount = 128
+	const replacementCount = 32
+
+	var policies collectionPolicyScope
+	collectionEntries := make([]document.Entry, 0, policyCount)
+	for index := 0; index < policyCount; index++ {
+		key := fmt.Sprintf("collection-%03d", index)
+		path := Keys("collections", key)
+		policies.set(path, collectionPolicy{identityFields: []string{"id"}, identityPresent: true})
+		collectionEntries = append(collectionEntries, document.Entry{
+			Key: key, Value: &document.Node{Kind: document.Sequence},
+		})
+	}
+
+	baseScalars := make([]document.Entry, 0, replacementCount)
+	incomingScalars := make([]document.Entry, 0, replacementCount)
+	for index := 0; index < replacementCount; index++ {
+		key := fmt.Sprintf("scalar-%03d", index)
+		baseScalars = append(baseScalars, document.Entry{
+			Key: key, Value: &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+				Key: "previous", Value: &document.Node{Kind: document.Boolean, Text: "true"},
+			}}},
+		})
+		incomingScalars = append(incomingScalars, document.Entry{
+			Key: key, Value: &document.Node{Kind: document.String, Text: "replaced"},
+		})
+	}
+	base := &composedDocument{
+		root: &document.Node{Kind: document.Mapping, Mapping: []document.Entry{
+			{Key: "collections", Value: &document.Node{Kind: document.Mapping, Mapping: collectionEntries}},
+			{Key: "scalars", Value: &document.Node{Kind: document.Mapping, Mapping: baseScalars}},
+		}},
+		collectionPolicies: policies,
+		origins:            make(map[string]Origin),
+	}
+	incoming := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+		Key: "scalars", Value: &document.Node{Kind: document.Mapping, Mapping: incomingScalars},
+	}}}
+	assembly := newAssemblyAccumulator(base)
+	if _, diagnostics := assembly.applyOverlayWithCollectionProjection(
+		incoming, nil, collectionPolicyScope{}, nil, LayerOverlay, sequencePolicy{}, nil,
+	); len(diagnostics) != 0 {
+		t.Fatalf("apply disjoint scalar replacements: %#v", diagnostics)
+	}
+	for index := 0; index < policyCount; index++ {
+		path := Keys("collections", fmt.Sprintf("collection-%03d", index))
+		policy, found := assembly.current.collectionPolicies.at(path)
+		if !found || !policy.identityPresent || len(policy.identityFields) != 1 || policy.identityFields[0] != "id" {
+			t.Fatalf("disjoint replacement changed policy %s: %#v, found %t", path, policy, found)
+		}
+	}
+	for index := 0; index < replacementCount; index++ {
+		path := Keys("scalars", fmt.Sprintf("scalar-%03d", index))
+		value, found := documentAtPath(assembly.current.root, path)
+		if !found || value.Kind != document.String || value.Text != "replaced" {
+			t.Fatalf("scalar replacement at %s = %#v, found %t", path, value, found)
+		}
+	}
+}

internal/engine/compose.go

@@ -45,10 +45,18 @@ type sequenceHistoryContribution struct {
 type assemblyAccumulator struct {
 	current                    *composedDocument
 	index                      mappingIndex
+	collectionPolicyRetirement *collectionPolicyRetirementIndex
 }
 
 func newAssemblyAccumulator(owned *composedDocument) *assemblyAccumulator {
-	return &assemblyAccumulator{current: owned}
+	policies := collectionPolicyScope{}
+	if owned != nil {
+		policies = owned.collectionPolicies
+	}
+	return &assemblyAccumulator{
+		current:                    owned,
+		collectionPolicyRetirement: newCollectionPolicyRetirementIndex(policies),
+	}
 }
 
 // cloneComposedDocument detaches mutable composition and provenance state;

internal/engine/overlay.go

@@ -260,7 +260,7 @@ func (operation *overlayMergeOperation) combined(path Path, incoming *document.N
 }
 
 func (operation *overlayMergeOperation) replaced(path Path, existing, incoming *document.Node) {
-	operation.result.collectionPolicies.retireCollectionPoliciesAtOrBelow(path)
+	operation.assembly.retireCollectionPoliciesAtOrBelow(path)
 	recordOverlayReplacement(operation.result, path, existing, incoming, operation.layer, true, operation.incomingOrigins)
 }
 
@@ -275,7 +275,7 @@ func (operation *overlayMergeOperation) merge(path Path, existing, incoming *doc
 		return existing, false
 	}
 	applyOverlaySequenceEvidence(
-		operation.result, path, existing, incoming, combined, resolution,
+		operation.assembly, path, existing, incoming, combined, resolution,
 		operation.incomingOrigins, operation.incomingPolicies, operation.layer,
 	)
 	operation.valueDestinations[path.String()] = true
@@ -334,7 +334,7 @@ func finalizeOverlayCollection(
 			diagnostics = append(diagnostics, issues...)
 			continue
 		}
-		assembly.current.collectionPolicies.set(path, policy)
+		assembly.setCollectionPolicy(path, policy)
 	}
 	if len(diagnostics) != 0 {
 		return nil, canonicalDiagnostics(diagnostics)
@@ -414,7 +414,7 @@ func recordOverlayReplacement(
 }
 
 func applyOverlaySequenceEvidence(
-	result *composedDocument,
+	assembly *assemblyAccumulator,
 	path Path,
 	lower, higher *document.Node,
 	combined sequenceCombinationResult,
@@ -423,19 +423,20 @@ func applyOverlaySequenceEvidence(
 	incomingPolicies collectionPolicyScope,
 	layer Layer,
 ) {
+	result := assembly.current
 	lowerPolicy, hadLowerPolicy := result.collectionPolicies.at(path)
 	if resolution.operation == sequenceModeReplace {
 		var lowerValue *collectionPolicy
 		if hadLowerPolicy {
 			lowerValue = &lowerPolicy
 		}
-		result.collectionPolicies.retireCollectionPoliciesAtOrBelow(path)
+		assembly.retireCollectionPoliciesAtOrBelow(path)
 		resolution.policy = collectionPolicyForReplacement(lowerValue, resolution.policy)
 	}
 
 	if combined.value == nil || combined.value.Kind != document.Sequence {
 		recordOverlayReplacement(result, path, lower, higher, layer, true, incomingOrigins)
-		result.collectionPolicies.set(path, resolution.policy)
+		assembly.setCollectionPolicy(path, resolution.policy)
 		return
 	}
 
@@ -519,7 +520,7 @@ func applyOverlaySequenceEvidence(
 			policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy())
 		}
 	}
-	result.collectionPolicies.set(path, policy)
+	assembly.setCollectionPolicy(path, policy)
 }
 
 func collectOverlayOrigins(origins map[string]Origin, node *document.Node, path Path, layer Layer) {