Prepare collection policy transport through inheritance

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Sequence Identity And Combination Implementation Plan
 
-Status: implementation in progress; Stages 1–16 complete.
+Status: implementation in progress; Stages 1–17 complete.
 
 Audience: a Luna coding agent implementing one numbered stage per prompt, in
 order. This plan starts at commit
@@ -613,6 +613,11 @@ collection-aware inheritance operations in Stage 18.
 Validation: existing inheritance, ownership, history, and resource-context tests,
 focused transport tests, and the everyday check.
 
+**Completion:** Implemented detached collection-policy transport through
+inheritance templates and expansion operations while preserving current values
+and nested instructions. Verified focused inheritance/history/resource tests,
+`go test -race ./internal/engine`, and `./scripts/check.sh`.
+
 ## Stage 18 — Apply Collection Policy During Mapping-Sibling Inheritance
 
 Scope: activate shared collection operations using the prepared template/state

internal/engine/inheritance.go

@@ -7,10 +7,13 @@ const inheritanceKey = "extends"
 // These private records keep structural evidence separate from acquired source
 // records. Later inspection can project a transfer onto individual descendants.
 type inheritanceTransfer struct {
-	from      Path
+	from              Path
-	to        Path
+	to                Path
-	reference Path
+	reference         Path
-	locations []Location
+	locations         []Location
+	policyTemplates   []collectionPolicyScope
+	templatePolicies  collectionPolicyScope
+	projectedPolicies []projectedCollectionPolicy
 }
 
 type inheritanceControl struct {
@@ -20,22 +23,125 @@ type inheritanceControl struct {
 }
 
 type inheritanceScope struct {
-	root         *document.Node
+	root               *document.Node
-	origins      map[string]Origin
+	origins            map[string]Origin
-	transfers    []inheritanceTransfer
+	collectionPolicies collectionPolicyScope
-	controls     []inheritanceControl
+	transfers          []inheritanceTransfer
-	replacements []inheritanceReplacement
+	controls           []inheritanceControl
-	operations   []inheritanceOperation
+	replacements       []inheritanceReplacement
+	operations         []inheritanceOperation
 }
 
 type inheritanceOperation struct {
-	control          *inheritanceControl
+	control           *inheritanceControl
-	transfer         *inheritanceTransfer
+	transfer          *inheritanceTransfer
-	controlOrigin    Origin
+	controlOrigin     Origin
-	parent           *document.Node
+	parent            *document.Node
-	effective        *document.Node
+	effective         *document.Node
-	parentOrigins    map[string]Origin
+	parentOrigins     map[string]Origin
-	effectiveOrigins map[string]Origin
+	effectiveOrigins  map[string]Origin
+	parentPolicies    []collectionPolicyScope
+	templatePolicies  collectionPolicyScope
+	effectivePolicies []collectionPolicyScope
+}
+
+func cloneInheritancePolicyTemplates(templates []collectionPolicyScope) []collectionPolicyScope {
+	if templates == nil {
+		return nil
+	}
+	cloned := make([]collectionPolicyScope, len(templates))
+	for index, template := range templates {
+		cloned[index] = cloneCollectionPolicyScopeForInheritance(template)
+	}
+	return cloned
+}
+
+func cloneProjectedInheritancePolicies(policies []projectedCollectionPolicy) []projectedCollectionPolicy {
+	if policies == nil {
+		return nil
+	}
+	cloned := make([]projectedCollectionPolicy, len(policies))
+	for index, policy := range policies {
+		cloned[index] = projectedCollectionPolicy{
+			path: policy.path.clone(), policy: cloneCollectionPolicyForInheritance(policy.policy),
+		}
+	}
+	return cloned
+}
+
+func cloneInheritanceTransfer(transfer inheritanceTransfer) inheritanceTransfer {
+	cloned := transfer
+	cloned.from = transfer.from.clone()
+	cloned.to = transfer.to.clone()
+	cloned.reference = transfer.reference.clone()
+	cloned.locations = append([]Location(nil), transfer.locations...)
+	cloned.policyTemplates = cloneInheritancePolicyTemplates(transfer.policyTemplates)
+	cloned.templatePolicies = cloneCollectionPolicyScopeForInheritance(transfer.templatePolicies)
+	cloned.projectedPolicies = cloneProjectedInheritancePolicies(transfer.projectedPolicies)
+	return cloned
+}
+
+func cloneInheritanceOperations(operations []inheritanceOperation) []inheritanceOperation {
+	if operations == nil {
+		return nil
+	}
+	cloned := make([]inheritanceOperation, len(operations))
+	for index, operation := range operations {
+		cloned[index] = operation
+		if operation.control != nil {
+			control := *operation.control
+			control.path = operation.control.path.clone()
+			control.locations = append([]Location(nil), operation.control.locations...)
+			cloned[index].control = &control
+		}
+		if operation.transfer != nil {
+			transfer := cloneInheritanceTransfer(*operation.transfer)
+			cloned[index].transfer = &transfer
+		}
+		cloned[index].parent = document.Clone(operation.parent)
+		cloned[index].effective = document.Clone(operation.effective)
+		cloned[index].parentOrigins = cloneOriginIndex(operation.parentOrigins)
+		cloned[index].effectiveOrigins = cloneOriginIndex(operation.effectiveOrigins)
+		cloned[index].parentPolicies = cloneInheritancePolicyTemplates(operation.parentPolicies)
+		cloned[index].templatePolicies = cloneCollectionPolicyScopeForInheritance(operation.templatePolicies)
+		cloned[index].effectivePolicies = cloneInheritancePolicyTemplates(operation.effectivePolicies)
+	}
+	return cloned
+}
+
+func inheritancePolicyScopeAtPath(scope collectionPolicyScope, prefix Path) collectionPolicyScope {
+	localized := collectionPolicyScope{}
+	for _, entry := range scope.collections {
+		if relative, inside := relativeHistoryPath(prefix, entry.destination); inside {
+			localized.set(relative, cloneCollectionPolicyForInheritance(entry.policy))
+			continue
+		}
+		relative, inside := relativeHistoryPath(entry.destination, prefix)
+		if !inside || len(relative.segments) == 0 || relative.segments[0].kind != pathSegmentIndex {
+			continue
+		}
+		record, found := entry.policy.recordPolicyAt(relative.segments[0].index)
+		if !found {
+			continue
+		}
+		remainder := Path{segments: append([]pathSegment(nil), relative.segments[1:]...)}
+		for _, nested := range inheritancePolicyScopeAtPath(record.scope, remainder).collections {
+			localized.set(nested.destination, cloneCollectionPolicyForInheritance(nested.policy))
+		}
+	}
+	return localized
+}
+
+func projectInheritancePolicyTemplates(templates []collectionPolicyScope, prefix Path) []projectedCollectionPolicy {
+	var projected []projectedCollectionPolicy
+	for _, template := range templates {
+		for _, policy := range projectCollectionPolicyScope(template, prefix) {
+			projected = append(projected, projectedCollectionPolicy{
+				path: policy.path.clone(), policy: cloneCollectionPolicyForInheritance(policy.policy),
+			})
+		}
+	}
+	return projected
 }
 
 // A replacement retains the displaced source/derivation record while the

internal/engine/inheritance_history.go

@@ -73,7 +73,7 @@ func applyInheritanceMetadata(metadata *Metadata, raw *document.Node, rawOrigins
 	})
 	applyOverlayMetadata(&sourceView, overlays)
 	metadata.history = sourceView.history
-	metadata.inheritance = append([]inheritanceOperation(nil), expansion.operations...)
+	metadata.inheritance = cloneInheritanceOperations(expansion.operations)
 }
 
 func historyWithInheritance(path Path, stored []Change, operations []inheritanceOperation) []Change {

internal/engine/inheritance_load.go

@@ -63,5 +63,5 @@ func inheritanceOperations(expanded *inheritanceScope) []inheritanceOperation {
 	if expanded == nil {
 		return nil
 	}
-	return expanded.operations
+	return cloneInheritanceOperations(expanded.operations)
 }

internal/engine/inheritance_schedule.go

@@ -7,14 +7,15 @@ import (
 )
 
 type inheritanceTemplate struct {
-	name         string
+	name               string
-	entry        document.Entry
+	entry              document.Entry
-	path         Path
+	path               Path
-	parent       int
+	collectionPolicies collectionPolicyScope
-	hasParent    bool
+	parent             int
-	hasDirective bool
+	hasParent          bool
-	cancelled    bool
+	hasDirective       bool
-	directive    document.Entry
+	cancelled          bool
+	directive          document.Entry
 }
 
 type inheritanceGraph struct {
@@ -29,6 +30,10 @@ type inheritanceGraph struct {
 // It validates declarations separately from applying their templates, leaving
 // nested scopes for the existing destination-local walk.
 func analyzeInheritanceGraph(scope *document.Node, scopePath Path) (*inheritanceGraph, []Diagnostic) {
+	return analyzeInheritanceGraphWithPolicies(scope, scopePath, collectionPolicyScope{})
+}
+
+func analyzeInheritanceGraphWithPolicies(scope *document.Node, scopePath Path, policies collectionPolicyScope) (*inheritanceGraph, []Diagnostic) {
 	if scope == nil || scope.Kind != document.Mapping {
 		return nil, []Diagnostic{{
 			Code: CodeInheritanceInvalid, Severity: SeverityError, Path: scopePath,
@@ -57,6 +62,7 @@ func analyzeInheritanceGraph(scope *document.Node, scopePath Path) (*inheritance
 		graph.byName[name] = index
 		graph.nodes = append(graph.nodes, inheritanceTemplate{
 			name: name, entry: entries[name], path: scopePath.Key(name),
+			collectionPolicies: inheritancePolicyScopeAtPath(policies, scopePath.Key(name)),
 		})
 	}
 
@@ -115,7 +121,11 @@ func (graph *inheritanceGraph) schedule() ([]int, []Diagnostic) {
 }
 
 func expandSiblingDefinitionsWithOrigins(scope *document.Node, scopePath Path, limits projectionLimits, inputOrigins map[string]Origin, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
-	graph, diagnostics := analyzeInheritanceGraph(scope, scopePath)
+	return expandSiblingDefinitionsWithPolicies(scope, scopePath, limits, inputOrigins, collectionPolicyScope{}, budget...)
+}
+
+func expandSiblingDefinitionsWithPolicies(scope *document.Node, scopePath Path, limits projectionLimits, inputOrigins map[string]Origin, policies collectionPolicyScope, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
+	graph, diagnostics := analyzeInheritanceGraphWithPolicies(scope, scopePath, policies)
 	if len(diagnostics) != 0 {
 		return nil, diagnostics
 	}
@@ -123,7 +133,12 @@ func expandSiblingDefinitionsWithOrigins(scope *document.Node, scopePath Path, l
 	if len(diagnostics) != 0 {
 		return nil, diagnostics
 	}
-	return applyInheritanceTemplates(graph, order, limits, inputOrigins, budget...)
+	result, diagnostics := applyInheritanceTemplates(graph, order, limits, inputOrigins, budget...)
+	if len(diagnostics) != 0 {
+		return nil, diagnostics
+	}
+	result.collectionPolicies = cloneCollectionPolicyScopeForInheritance(policies)
+	return result, nil
 }
 
 // applyInheritanceTemplates builds detached templates in the supplied
@@ -132,6 +147,7 @@ func expandSiblingDefinitionsWithOrigins(scope *document.Node, scopePath Path, l
 func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits projectionLimits, inputOrigins map[string]Origin, budget ...*retainedNodeCounter) (*inheritanceScope, []Diagnostic) {
 	resolved := make([]*document.Node, len(graph.nodes))
 	resolvedOrigins := make([]map[string]Origin, len(graph.nodes))
+	resolvedPolicyTemplates := make([][]collectionPolicyScope, len(graph.nodes))
 	result := &inheritanceScope{}
 	counter := newRetainedNodeCounter(historyNodeLimit)
 	if len(budget) != 0 {
@@ -155,6 +171,7 @@ func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits proj
 
 	for _, index := range order {
 		node := graph.nodes[index]
+		templatePolicies := cloneCollectionPolicyScopeForInheritance(node.collectionPolicies)
 		payload := withoutInheritanceDirective(node.entry.Value)
 		operationStart, replacementStart := len(result.operations), len(result.replacements)
 		childOrigins := directInheritanceOrigins(payload, node.path, inputOrigins)
@@ -183,11 +200,15 @@ func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits proj
 
 		if node.hasParent {
 			parent := resolved[node.parent]
+			parentPolicies := cloneInheritancePolicyTemplates(resolvedPolicyTemplates[node.parent])
+			effectivePolicies := append(cloneInheritancePolicyTemplates(parentPolicies), cloneCollectionPolicyScopeForInheritance(templatePolicies))
 			merged := document.Clone(parent)
 			transfer := inheritanceTransfer{
 				from: graph.nodes[node.parent].path, to: node.path,
-				reference: node.path.Key(inheritanceKey),
+				reference:       node.path.Key(inheritanceKey),
-				locations: []Location{publicLocation(node.directive.Value.Location)},
+				locations:       []Location{publicLocation(node.directive.Value.Location)},
+				policyTemplates: parentPolicies, templatePolicies: cloneCollectionPolicyScopeForInheritance(templatePolicies),
+				projectedPolicies: projectInheritancePolicyTemplates(parentPolicies, node.path),
 			}
 			mergedOrigins := transferredInheritanceOrigins(resolvedOrigins[node.parent], transfer.from, node.path, transfer)
 			mergeInheritanceOrigin(mergedOrigins, node.path, childOrigins[node.path.String()])
@@ -214,10 +235,15 @@ func applyInheritanceTemplates(graph *inheritanceGraph, order []int, limits proj
 			result.operations = append(result.operations, inheritanceOperation{
 				transfer: &transfer, parent: parent, effective: merged,
 				parentOrigins: resolvedOrigins[node.parent], effectiveOrigins: mergedOrigins,
+				parentPolicies:    cloneInheritancePolicyTemplates(parentPolicies),
+				templatePolicies:  cloneCollectionPolicyScopeForInheritance(templatePolicies),
+				effectivePolicies: cloneInheritancePolicyTemplates(effectivePolicies),
 			})
+			resolvedPolicyTemplates[index] = effectivePolicies
 		} else {
 			resolved[index] = document.Clone(payload)
 			resolvedOrigins[index] = childOrigins
+			resolvedPolicyTemplates[index] = []collectionPolicyScope{templatePolicies}
 		}
 
 		count, _ := inheritanceLowerBoundCount(resolved[index], limits.nodes)

internal/engine/inheritance_schedule_test.go

@@ -110,6 +110,91 @@ beta:
 	}
 }
 
+func TestInheritanceTransportsReusableCollectionPolicyTemplates(t *testing.T) {
+	root := parseItemPolicyDocument(t, "templates", `base:
+  prompts:
+    - id: shared
+      children:
+        derived:
+          extends: nested-base
+          own: destination-local
+        nested-base:
+          inherited: source-value
+first:
+  extends: base
+second:
+  extends: base
+`)
+
+	policyLocation := Location{SourceID: "base-resource", Line: 2, Column: 3}
+	nestedLocation := Location{SourceID: "nested-resource", Line: 5, Column: 11}
+	nested := collectionPolicy{identityFields: []string{"key"}, identityPresent: true, identityLocation: &nestedLocation}
+	recordScope := collectionPolicyScope{}
+	recordScope.set(Keys("children", "prompts"), nested)
+	basePolicy := collectionPolicy{identityFields: []string{"id"}, identityPresent: true, identityLocation: &policyLocation}
+	basePolicy.setRecordPolicyAt(0, collectionRecordPolicy{owner: newCollectionRecordPolicy().owner, scope: recordScope})
+	policies := collectionPolicyScope{}
+	policies.set(Keys("base", "prompts"), basePolicy)
+
+	got, diagnostics := expandSiblingDefinitionsWithPolicies(root, Path{}, defaultProjectionLimits(), nil, policies)
+	if len(diagnostics) != 0 {
+		t.Fatalf("expand sibling templates: %#v", diagnostics)
+	}
+	withoutPolicies, diagnostics := expandSiblingDefinitionsWithOrigins(root, Path{}, defaultProjectionLimits(), nil)
+	if len(diagnostics) != 0 {
+		t.Fatalf("expand sibling templates without policy: %#v", diagnostics)
+	}
+	if !reflect.DeepEqual(got.root, withoutPolicies.root) {
+		t.Fatalf("policy transport changed inherited values:\n got %#v\nwant %#v", got.root, withoutPolicies.root)
+	}
+	var transfers []inheritanceOperation
+	for _, operation := range got.operations {
+		if operation.transfer != nil {
+			transfers = append(transfers, operation)
+		}
+	}
+	if len(transfers) != 2 {
+		t.Fatalf("transfer operations = %d, want two destination transfers", len(transfers))
+	}
+
+	for index, destination := range []string{"/first", "/second"} {
+		operation := transfers[index]
+		if operation.transfer == nil || len(operation.transfer.projectedPolicies) != 2 {
+			t.Fatalf("transfer %d projected policies = %#v, want collection and record-local policy", index, operation.transfer)
+		}
+		projected := operation.transfer.projectedPolicies
+		if projected[0].path.String() != destination+"/prompts" || projected[1].path.String() != destination+"/prompts/0/children/prompts" {
+			t.Fatalf("transfer %d policy paths = %q, %q", index, projected[0].path, projected[1].path)
+		}
+		if got := projected[0].policy.identityLocation; got == nil || *got != policyLocation {
+			t.Fatalf("transfer %d collection declaration location = %#v", index, got)
+		}
+		if got := projected[1].policy.identityLocation; got == nil || *got != nestedLocation {
+			t.Fatalf("transfer %d record-local declaration location = %#v", index, got)
+		}
+		if len(operation.templatePolicies.collections) != 0 {
+			t.Fatalf("transfer %d unexpectedly copied parent YAML policy onto child template", index)
+		}
+	}
+
+	first, second := transfers[0].transfer.projectedPolicies[0].policy, transfers[1].transfer.projectedPolicies[0].policy
+	if first.recordOrder[0] == second.recordOrder[0] {
+		t.Fatal("destination policy templates share a mutable record owner")
+	}
+	first.identityFields[0] = "changed"
+	if second.identityFields[0] != "id" {
+		t.Fatalf("mutating one destination changed another: %v", second.identityFields)
+	}
+	if basePolicy.identityFields[0] != "id" {
+		t.Fatalf("mutating a destination changed the source template: %v", basePolicy.identityFields)
+	}
+
+	firstValue, found := documentAtPath(got.root.Mapping[1].Value, Keys("prompts").Index(0).Key("children").Key("derived").Key("extends"))
+	if !found || firstValue.Text != "nested-base" {
+		t.Fatalf("nested template instruction = %#v, found %t; want original instruction retained", firstValue, found)
+	}
+}
+
 func TestInheritanceExpansionInterleavesUnequalBranches(t *testing.T) {
 	root := parseCompositionDocument(t, "branches", `a-grandchild:
   extends: z-child

internal/engine/inheritance_walk.go

@@ -8,14 +8,15 @@ import (
 )
 
 type inheritanceWalker struct {
-	nodes        int // logical non-directive nodes in the entire working tree
+	nodes              int // logical non-directive nodes in the entire working tree
-	origins      map[string]Origin
+	origins            map[string]Origin
-	limits       projectionLimits
+	collectionPolicies collectionPolicyScope
-	retained     *retainedNodeCounter
+	limits             projectionLimits
-	transfers    []inheritanceTransfer
+	retained           *retainedNodeCounter
-	controls     []inheritanceControl
+	transfers          []inheritanceTransfer
-	replacements []inheritanceReplacement
+	controls           []inheritanceControl
-	operations   []inheritanceOperation
+	replacements       []inheritanceReplacement
+	operations         []inheritanceOperation
 }
 
 // expandInheritanceTree resolves each destination scope after its enclosing
@@ -25,10 +26,18 @@ func expandInheritanceTree(input *document.Node, limits projectionLimits) (*inhe
 }
 
 func expandInheritanceWithOrigins(input *document.Node, inputOrigins map[string]Origin, limits projectionLimits) (*inheritanceScope, []Diagnostic) {
-	return expandInheritanceWithBudgets(input, inputOrigins, limits, historyNodeLimit)
+	return expandInheritanceWithPolicies(input, inputOrigins, collectionPolicyScope{}, limits)
 }
 
 func expandInheritanceWithBudgets(input *document.Node, inputOrigins map[string]Origin, limits projectionLimits, historyLimit int) (*inheritanceScope, []Diagnostic) {
+	return expandInheritanceWithPolicyBudgets(input, inputOrigins, collectionPolicyScope{}, limits, historyLimit)
+}
+
+func expandInheritanceWithPolicies(input *document.Node, inputOrigins map[string]Origin, policies collectionPolicyScope, limits projectionLimits) (*inheritanceScope, []Diagnostic) {
+	return expandInheritanceWithPolicyBudgets(input, inputOrigins, policies, limits, historyNodeLimit)
+}
+
+func expandInheritanceWithPolicyBudgets(input *document.Node, inputOrigins map[string]Origin, policies collectionPolicyScope, limits projectionLimits, historyLimit int) (*inheritanceScope, []Diagnostic) {
 	if input == nil || input.Kind != document.Mapping {
 		return nil, []Diagnostic{{Code: CodeInheritanceInvalid, Severity: SeverityError,
 			Message: "inheritance requires a mapping document root"}}
@@ -44,7 +53,8 @@ func expandInheritanceWithBudgets(input *document.Node, inputOrigins map[string]
 	}
 	walker := inheritanceWalker{
 		nodes: nodes, limits: limits, origins: origins,
-		retained: newRetainedNodeCounter(historyLimit),
+		collectionPolicies: cloneCollectionPolicyScopeForInheritance(policies),
+		retained:           newRetainedNodeCounter(historyLimit),
 	}
 	if problem := walker.visit(&root, Path{}, true); problem != nil {
 		return nil, []Diagnostic{*problem}
@@ -53,8 +63,10 @@ func expandInheritanceWithBudgets(input *document.Node, inputOrigins map[string]
 		return nil, []Diagnostic{*problem}
 	}
 	return &inheritanceScope{
-		root: root, origins: walker.origins, transfers: walker.transfers,
+		root: root, origins: walker.origins,
-		controls: walker.controls, replacements: walker.replacements, operations: walker.operations,
+		collectionPolicies: cloneCollectionPolicyScopeForInheritance(walker.collectionPolicies),
+		transfers:          walker.transfers,
+		controls:           walker.controls, replacements: walker.replacements, operations: walker.operations,
 	}, nil
 }
 
@@ -90,7 +102,7 @@ func (w *inheritanceWalker) visit(node **document.Node, path Path, unscoped bool
 			if limits.nodes >= 0 {
 				limits.nodes -= w.nodes - before
 			}
-			resolved, diagnostics := expandSiblingDefinitionsWithOrigins(*node, path, limits, w.origins, w.retained)
+			resolved, diagnostics := expandSiblingDefinitionsWithPolicies(*node, path, limits, w.origins, w.collectionPolicies, w.retained)
 			if len(diagnostics) != 0 {
 				return &diagnostics[0]
 			}

internal/engine/load.go

@@ -258,7 +258,9 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
 			accumulator.add(problems...)
 			return nil, accumulator.loadError()
 		}
-		expanded, problems := expandInheritanceWithOrigins(composition.root, composition.origins, defaultProjectionLimits())
+		expanded, problems := expandInheritanceWithPolicies(
+			composition.root, composition.origins, composition.collectionPolicies, defaultProjectionLimits(),
+		)
 		if len(problems) != 0 {
 			accumulator.add(problems...)
 			return nil, accumulator.loadError()