Apply sparse collection policy across overlays

docs/roadmap/implementation.md

@@ -945,6 +945,14 @@ record owners, plus retained-key-only/mode-only and ancestor-replacement cases.
 Required value/history copies and affected record payload remain legitimate costs.
 Focused check: `go test ./internal/engine -run 'Collection|Retirement|Sequence|Underlay|Layered|Inheritance|History'`.
 
+Status: **complete** (E02). Prior completed commits include `8832449`,
+`549fca2`, `68383f3`, and `3a1b6ff`. Validation passed:
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'Collection|Retirement|Sequence|Underlay|Layered|Inheritance|History' -count=1`,
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./... -count=1`,
+`env GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test -race ./...`,
+and `git diff --check`.
+
 ## Stage 29 — Consolidate Sequence-History Translation And Finalize Policy Owners
 
 Finding: completes **E03**.

internal/engine/collection_retirement_test.go

@@ -241,3 +241,110 @@ func TestDisjointScalarReplacementsRetainIndexedCollectionPolicies(t *testing.T)
 		}
 	}
 }
+
+func TestDisjointTierOverlaysRetainNestedCollectionPolicies(t *testing.T) {
+	t.Parallel()
+
+	nestedOwner := newCollectionRecordPolicy()
+	nestedOwner.scope.set(Keys("prompts"), collectionPolicy{
+		identityFields: []string{"name"}, identityPresent: true,
+		explicitMode: sequenceModeAppend, explicitModePresent: true,
+	})
+	itemsPolicy := collectionPolicyWithRecords(collectionPolicy{
+		identityFields: []string{"id"}, identityPresent: true,
+		explicitMode: sequenceModeAppend, explicitModePresent: true,
+	}, nestedOwner)
+	var policies collectionPolicyScope
+	policies.set(Keys("items"), itemsPolicy)
+	policies.set(Keys("keyOnly"), collectionPolicy{
+		identityFields: []string{"id"}, identityPresent: true,
+	})
+	policies.set(Keys("modeOnly"), collectionPolicy{
+		explicitMode: sequenceModeAppend, explicitModePresent: true,
+	})
+
+	base := &composedDocument{
+		root: &document.Node{Kind: document.Mapping, Mapping: []document.Entry{
+			{Key: "items", Value: &document.Node{Kind: document.Sequence, Sequence: []*document.Node{{
+				Kind: document.Mapping,
+				Mapping: []document.Entry{
+					{Key: "id", Value: &document.Node{Kind: document.String, Text: "one"}},
+					{Key: "prompts", Value: &document.Node{Kind: document.Sequence}},
+				},
+			}}}},
+			{Key: "keyOnly", Value: &document.Node{Kind: document.Sequence}},
+			{Key: "modeOnly", Value: &document.Node{Kind: document.Sequence}},
+		}},
+		collectionPolicies: policies,
+		origins:            make(map[string]Origin),
+	}
+	assembly := newAssemblyAccumulator(base)
+	initialItems, found := assembly.current.collectionPolicies.at(Keys("items"))
+	if !found {
+		t.Fatal("initial item policy is missing")
+	}
+	initialRecord, found := initialItems.recordPolicyAt(0)
+	if !found {
+		t.Fatal("initial nested record owner is missing")
+	}
+
+	for index, key := range []string{"firstTier", "secondTier", "thirdTier"} {
+		overlay := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+			Key: key, Value: &document.Node{Kind: document.Boolean, Text: "true"},
+		}}}
+		if _, diagnostics := assembly.applyOverlayWithCollectionProjection(
+			overlay, nil, collectionPolicyScope{}, nil, LayerUnderlay, sequencePolicy{}, nil,
+		); len(diagnostics) != 0 {
+			t.Fatalf("apply disjoint tier %d: %#v", index, diagnostics)
+		}
+
+		retainedItems, found := assembly.current.collectionPolicies.at(Keys("items"))
+		if !found || !retainedItems.identityPresent || len(retainedItems.identityFields) != 1 || retainedItems.identityFields[0] != "id" ||
+			!retainedItems.explicitModePresent || retainedItems.explicitMode != sequenceModeAppend {
+			t.Fatalf("tier %d changed retained item policy: %#v, found %t", index, retainedItems, found)
+		}
+		retainedRecord, found := retainedItems.recordPolicyAt(0)
+		if !found || retainedRecord.owner != initialRecord.owner {
+			t.Fatalf("tier %d replaced the nested record owner: %#v, found %t", index, retainedRecord, found)
+		}
+		nested, found := retainedRecord.scope.at(Keys("prompts"))
+		if !found || !nested.identityPresent || len(nested.identityFields) != 1 || nested.identityFields[0] != "name" ||
+			!nested.explicitModePresent || nested.explicitMode != sequenceModeAppend {
+			t.Fatalf("tier %d changed nested collection policy: %#v, found %t", index, nested, found)
+		}
+		keyOnly, found := assembly.current.collectionPolicies.at(Keys("keyOnly"))
+		if !found || !keyOnly.identityPresent || keyOnly.explicitModePresent {
+			t.Fatalf("tier %d changed key-only policy: %#v, found %t", index, keyOnly, found)
+		}
+		modeOnly, found := assembly.current.collectionPolicies.at(Keys("modeOnly"))
+		if !found || modeOnly.identityPresent || !modeOnly.explicitModePresent || modeOnly.explicitMode != sequenceModeAppend {
+			t.Fatalf("tier %d changed mode-only policy: %#v, found %t", index, modeOnly, found)
+		}
+	}
+}
+
+func TestOverlayNonSequenceClaimAtRetainedCollectionStillFails(t *testing.T) {
+	t.Parallel()
+
+	var policies collectionPolicyScope
+	policies.set(Keys("items"), collectionPolicy{
+		identityFields: []string{"id"}, identityPresent: true,
+		explicitMode: sequenceModeAppend, explicitModePresent: true,
+	})
+	base := &composedDocument{
+		root: &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+			Key: "items", Value: &document.Node{Kind: document.Sequence},
+		}}},
+		collectionPolicies: policies,
+		origins:            make(map[string]Origin),
+	}
+	assembly := newAssemblyAccumulator(base)
+	overlay := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+		Key: "items", Value: &document.Node{Kind: document.String, Text: "not a sequence"},
+	}}}
+	if _, diagnostics := assembly.applyOverlayWithCollectionProjection(
+		overlay, nil, collectionPolicyScope{}, nil, LayerUnderlay, sequencePolicy{}, nil,
+	); len(diagnostics) == 0 {
+		t.Fatal("non-sequence overlay at an append collection unexpectedly succeeded")
+	}
+}

internal/engine/overlay.go

@@ -155,26 +155,21 @@ func prepareOverlayCollection(
 	}
 	registered := make(map[string]bool)
 	for _, claim := range claims.ordered {
+		key := claim.path.String()
+		if _, hasLowerPolicy := lowerPolicies.collections[key]; hasLowerPolicy {
+			registered[key] = true
+			registeredPaths[key] = claim.path.clone()
+		}
 		if claim.kind == document.Sequence {
 			if _, ok := sequencePolicy.identityFieldsAt(claim.path); ok {
-				registered[claim.path.String()] = true
-				registeredPaths[claim.path.String()] = claim.path.clone()
+				registered[key] = true
+				registeredPaths[key] = claim.path.clone()
 			}
 		}
 	}
 	for key := range declarationsByPath {
 		registered[key] = true
 	}
-	var overlayLookup *documentLookup
-	if len(lowerPolicies.collections) != 0 {
-		overlayLookup = buildDocumentLookup(overlay)
-	}
-	for key := range lowerPolicies.collections {
-		if node, found := documentAtPathWithLookup(overlay, lowerPolicies.collections[key].destination, overlayLookup); found && node != nil {
-			registered[key] = true
-			registeredPaths[key] = lowerPolicies.collections[key].destination.clone()
-		}
-	}
 
 	resolutions := make(map[string]collectionPolicyResolution, len(registered))
 	for key := range registered {