Preserve nested collection policy ownership

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Sequence Identity And Combination Implementation Plan
 
-Status: implementation in progress; Stages 1–15 complete.
+Status: implementation in progress; Stages 1–16 complete.
 
 Audience: a Luna coding agent implementing one numbered stage per prompt, in
 order. This plan starts at commit
@@ -586,6 +586,12 @@ Scope: the consequential lifetime cases across the now-active source assembly.
 Validation: focused policy/sequence/layered/ownership tests,
 `go test -race ./internal/engine`, and the everyday check.
 
+**Completion:** Updated layered precedence to transfer incoming item-owned
+collection policies through sequence identity projections. Verified retained,
+upserted, and replaced records, ancestor retirement and reintroduction, renewed
+identity, and append-mode ownership. Validation: `go test -race ./internal/engine`
+and `./scripts/check.sh`.
+
 ## Stage 17 — Prepare Inheritance Policy And Template Transport
 
 Scope: behavior-preserving inheritance state transport, before activating

internal/engine/item_policy_test.go

@@ -2,6 +2,7 @@ package engine
 
 import (
 	"fmt"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -194,6 +195,161 @@ func TestNestedItemPolicyProjectionUsesStableOutputIndexes(t *testing.T) {
 	}
 }
 
+func TestOverlaySequenceTransfersIncomingItemPolicyOwnership(t *testing.T) {
+	path := Keys("records")
+	registered := mustCompileSequencePolicy(t, []sequenceRegistration{{path: path, identityField: "id"}})
+	makePolicies := func(childKey string) collectionPolicyScope {
+		local := collectionPolicyScope{}
+		local.set(Keys("steps"), collectionPolicy{
+			identityFields: []string{childKey}, identityPresent: true,
+			explicitMode: sequenceModeAppend, explicitModePresent: true,
+		})
+		record := newCollectionRecordPolicy()
+		record.scope = local
+		outer := collectionPolicy{identityFields: []string{"id"}, identityPresent: true}
+		outer.setRecordPolicy(record)
+		scope := collectionPolicyScope{}
+		scope.set(path, outer)
+		return scope
+	}
+	checkPolicy := func(t *testing.T, scope collectionPolicyScope, index int, key string) {
+		t.Helper()
+		outer, ok := scope.at(path)
+		if !ok {
+			t.Fatal("outer collection policy missing")
+		}
+		record, ok := outer.recordPolicyAt(index)
+		if !ok {
+			t.Fatalf("record policy %d missing", index)
+		}
+		local, ok := record.scope.at(Keys("steps"))
+		if !ok || !local.identityPresent || !reflect.DeepEqual(local.identityFields, []string{key}) || !local.explicitModePresent || local.explicitMode != sequenceModeAppend {
+			t.Fatalf("record %d nested policy = %#v, %v; want key %q and append", index, local, ok, key)
+		}
+	}
+
+	tests := []struct {
+		name         string
+		lower        string
+		higher       string
+		declarations []sequenceDeclarationSpec
+		wantRecords  int
+	}{
+		{
+			name:        "upserting the same outer identity replaces its local policy",
+			lower:       "records: [{id: same, steps: [{id: old}]}]\n",
+			higher:      "records: [{id: same, steps: [{slug: new}]}]\n",
+			wantRecords: 1,
+		},
+		{
+			name:        "retained and appended outer records keep independent policies",
+			lower:       "records: [{id: lower, steps: [{id: old}]}]\n",
+			higher:      "records: [{id: higher, steps: [{slug: new}]}]\n",
+			wantRecords: 2,
+		},
+		{
+			name:   "whole sequence replacement transfers new local policy",
+			lower:  "records: [{id: same, steps: [{id: old}]}]\n",
+			higher: "records: [{id: same, steps: [{slug: new}]}]\n",
+			declarations: []sequenceDeclarationSpec{{
+				destination: path, mode: sequenceModeReplace, modePresent: true,
+			}},
+			wantRecords: 1,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			lowerRoot := parseItemPolicyDocument(t, "lower", test.lower)
+			incomingRoot := parseItemPolicyDocument(t, "higher", test.higher)
+			assembly := newAssemblyAccumulator(&composedDocument{
+				root: lowerRoot, collectionPolicies: makePolicies("id"), origins: make(map[string]Origin),
+			})
+			if diagnostics := assembly.applyOverlayWithCollectionPolicy(
+				incomingRoot, nil, makePolicies("slug"), nil, LayerBase, registered, test.declarations,
+			); len(diagnostics) != 0 {
+				t.Fatalf("overlay diagnostics = %#v", diagnostics)
+			}
+			outer, ok := assembly.current.collectionPolicies.at(path)
+			if !ok || len(outer.recordOrder) != test.wantRecords {
+				t.Fatalf("outer record policies = %#v, %v", outer, ok)
+			}
+			if test.wantRecords == 2 {
+				checkPolicy(t, assembly.current.collectionPolicies, 0, "id")
+				checkPolicy(t, assembly.current.collectionPolicies, 1, "slug")
+			} else {
+				checkPolicy(t, assembly.current.collectionPolicies, 0, "slug")
+			}
+		})
+	}
+}
+
+func TestAncestorReplacementRequiresIdentityWhenRetainedModeReturns(t *testing.T) {
+	path := Keys("catalog", "prompts")
+	newAssembly := func(t *testing.T) *assemblyAccumulator {
+		t.Helper()
+		policy := collectionPolicy{
+			identityFields: []string{"id"}, identityPresent: true,
+			explicitMode: sequenceModeAppend, explicitModePresent: true,
+		}
+		scope := collectionPolicyScope{}
+		scope.set(path, policy)
+		return newAssemblyAccumulator(&composedDocument{
+			root:               parseItemPolicyDocument(t, "base", "catalog:\n  prompts: [{id: old}]\n"),
+			collectionPolicies: scope, origins: make(map[string]Origin),
+		})
+	}
+	reintroduceAncestor := func(t *testing.T, assembly *assemblyAccumulator) {
+		t.Helper()
+		if diagnostics := assembly.applyOverlayWithCollectionPolicy(
+			parseItemPolicyDocument(t, "remove", "catalog: null\n"), nil, collectionPolicyScope{}, nil,
+			LayerBase, sequencePolicy{}, nil,
+		); len(diagnostics) != 0 {
+			t.Fatalf("ancestor removal diagnostics = %#v", diagnostics)
+		}
+		if diagnostics := assembly.applyOverlayWithCollectionPolicy(
+			parseItemPolicyDocument(t, "reintroduction", "catalog:\n  prompts: [{id: restored, name: restored}]\n"), nil,
+			collectionPolicyScope{}, nil, LayerBase, sequencePolicy{}, nil,
+		); len(diagnostics) != 0 {
+			t.Fatalf("ancestor reintroduction diagnostics = %#v", diagnostics)
+		}
+	}
+
+	t.Run("retained append mode requires a renewed identity", func(t *testing.T) {
+		assembly := newAssembly(t)
+		reintroduceAncestor(t, assembly)
+		policy, ok := assembly.current.collectionPolicies.at(path)
+		if !ok || policy.identityPresent || !policy.explicitModePresent || policy.explicitMode != sequenceModeAppend {
+			t.Fatalf("reintroduced policy = %#v, %v", policy, ok)
+		}
+		diagnostics := assembly.applyOverlayWithCollectionPolicy(
+			parseItemPolicyDocument(t, "next", "catalog:\n  prompts: [{value: next}]\n"), nil,
+			collectionPolicyScope{}, nil, LayerBase, sequencePolicy{}, nil,
+		)
+		if len(diagnostics) != 1 || diagnostics[0].Code != CodeSequenceInvalid {
+			t.Fatalf("append without renewed identity diagnostics = %#v", diagnostics)
+		}
+	})
+
+	t.Run("a renewed key restores the retained append mode", func(t *testing.T) {
+		assembly := newAssembly(t)
+		reintroduceAncestor(t, assembly)
+		declaration := sequenceDeclarationSpec{
+			destination: path, identityFields: []string{"name"}, identityPresent: true,
+		}
+		diagnostics := assembly.applyOverlayWithCollectionPolicy(
+			parseItemPolicyDocument(t, "keyed", "catalog:\n  prompts: [{name: next}]\n"), nil,
+			collectionPolicyScope{}, nil, LayerBase, sequencePolicy{}, []sequenceDeclarationSpec{declaration},
+		)
+		if len(diagnostics) != 0 {
+			t.Fatalf("renewed identity diagnostics = %#v", diagnostics)
+		}
+		policy, ok := assembly.current.collectionPolicies.at(path)
+		if !ok || !policy.identityPresent || !reflect.DeepEqual(policy.identityFields, []string{"name"}) || !policy.explicitModePresent || policy.explicitMode != sequenceModeAppend {
+			t.Fatalf("renewed collection policy = %#v, %v", policy, ok)
+		}
+	})
+}
+
 func TestItemSourceVersionProcessingConsumesDeclarationsBeforeComposition(t *testing.T) {
 	t.Parallel()
 

internal/engine/load_request.go

@@ -118,7 +118,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
 					declarations = append(declarations, acquiredSequenceDeclarations(scope)...)
 				}
 				if diagnostics := assembly.applyOverlayWithCollectionPolicy(
-					composition.root, composition.origins, composition.applications, LayerUnderlay,
+					composition.root, composition.origins, composition.collectionPolicies, composition.applications, LayerUnderlay,
 					request.sequencePolicy, declarations,
 				); len(diagnostics) != 0 {
 					return input, diagnostics
@@ -160,7 +160,7 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
 				declarations = append(declarations, acquiredSequenceDeclarations(scope)...)
 			}
 			if diagnostics := assembly.applyOverlayWithCollectionPolicy(
-				mainComposition.root, mainComposition.origins, mainComposition.applications, LayerBase,
+				mainComposition.root, mainComposition.origins, mainComposition.collectionPolicies, mainComposition.applications, LayerBase,
 				request.sequencePolicy, declarations,
 			); len(diagnostics) != 0 {
 				return input, diagnostics

internal/engine/overlay.go

@@ -59,6 +59,7 @@ func (a *assemblyAccumulator) applyOverlay(overlay *document.Node, incomingOrigi
 func (a *assemblyAccumulator) applyOverlayWithCollectionPolicy(
 	overlay *document.Node,
 	incomingOrigins map[string]Origin,
+	incomingPolicies collectionPolicyScope,
 	applications []historyApplication,
 	layer Layer,
 	sequencePolicy sequencePolicy,
@@ -68,7 +69,7 @@ func (a *assemblyAccumulator) applyOverlayWithCollectionPolicy(
 		return invalidOverlayDocument()
 	}
 	mergeOriginContribution(a.current.origins, Path{}, incomingOrigin(incomingOrigins, Path{}, overlay, layer))
-	sequences, diagnostics := mergeOverlayMappingsWithCollectionPolicy(a, overlay, incomingOrigins, layer, sequencePolicy, declarations)
+	sequences, diagnostics := mergeOverlayMappingsWithCollectionPolicy(a, overlay, incomingOrigins, incomingPolicies, layer, sequencePolicy, declarations)
 	if len(diagnostics) != 0 {
 		return diagnostics
 	}
@@ -89,6 +90,7 @@ func mergeOverlayMappingsWithCollectionPolicy(
 	assembly *assemblyAccumulator,
 	overlay *document.Node,
 	incomingOrigins map[string]Origin,
+	incomingPolicies collectionPolicyScope,
 	layer Layer,
 	sequencePolicy sequencePolicy,
 	declarations []sequenceDeclarationSpec,
@@ -172,7 +174,7 @@ func mergeOverlayMappingsWithCollectionPolicy(
 					combinationDiagnostics = append(combinationDiagnostics, issues...)
 					return existing, false
 				}
-				applyOverlaySequenceEvidence(result, path, existing, incoming, combined, resolution, incomingOrigins, layer)
+				applyOverlaySequenceEvidence(result, path, existing, incoming, combined, resolution, incomingOrigins, incomingPolicies, layer)
 				policyValues[path.String()] = true
 				history := sequenceHistoryContribution{path: path.clone(), mode: sequenceHistoryMapped}
 				if resolution.operation == sequenceModeReplace {
@@ -307,6 +309,7 @@ func applyOverlaySequenceEvidence(
 	combined sequenceCombinationResult,
 	resolution collectionPolicyResolution,
 	incomingOrigins map[string]Origin,
+	incomingPolicies collectionPolicyScope,
 	layer Layer,
 ) {
 	lowerPolicy, hadLowerPolicy := result.collectionPolicies.at(path)
@@ -383,22 +386,25 @@ func applyOverlaySequenceEvidence(
 	policy := cloneCollectionPolicy(resolution.policy)
 	policy.recordOrder = nil
 	policy.recordPolicies = nil
-	if hadLowerPolicy && resolution.operation != sequenceModeReplace {
+	lowerProjection := newSequenceItemProjectionIndex(combined.lowerItems)
-		lowerProjection := newSequenceItemProjectionIndex(combined.lowerItems)
+	higherProjection := newSequenceItemProjectionIndex(combined.higherItems)
-		higherProjection := newSequenceItemProjectionIndex(combined.higherItems)
+	incomingCollection, hasIncomingCollection := incomingPolicies.at(path)
-		for outputIndex := range combined.value.Sequence {
+	for outputIndex := range combined.value.Sequence {
-			if _, replaced := higherProjection.inputForOutput(outputIndex); replaced {
+		if higherIndex, replaced := higherProjection.inputForOutput(outputIndex); replaced {
+			if record, found := incomingCollection.recordPolicyAt(higherIndex); hasIncomingCollection && found {
+				policy.setRecordPolicyAt(outputIndex, retainedCollectionRecordPolicy(record))
+			} else {
 				policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy())
-				continue
 			}
+			continue
+		}
+		if hadLowerPolicy && resolution.operation != sequenceModeReplace {
 			if lowerIndex, retained := lowerProjection.inputForOutput(outputIndex); retained {
 				if record, found := lowerPolicy.recordPolicyAt(lowerIndex); found {
 					policy.setRecordPolicyAt(outputIndex, retainedCollectionRecordPolicy(record))
 				}
 			}
-		}
+		} else {
-	} else {
-		for outputIndex := range combined.value.Sequence {
 			policy.setRecordPolicyAt(outputIndex, replacedCollectionRecordPolicy())
 		}
 	}

internal/engine/selected_overlay.go

@@ -187,7 +187,7 @@ func loadSelectedOverlayWithAccessAndVersion(access *filesystemAccess, base *ass
 
 	var overlayDiagnostics []Diagnostic
 	if len(policies) != 0 {
-		overlayDiagnostics = base.applyOverlayWithCollectionPolicy(overlayPayload, nil, nil, LayerOverlay, policy, processed.sequences)
+		overlayDiagnostics = base.applyOverlayWithCollectionPolicy(overlayPayload, nil, collectionPolicyScope{}, nil, LayerOverlay, policy, processed.sequences)
 	} else {
 		overlayDiagnostics = base.applyOverlay(overlayPayload, nil, nil, LayerOverlay)
 	}