Separate underlay acquisition from composition

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Sequence Identity And Combination Implementation Plan
 
-Status: implementation in progress; Stages 1–9 complete.
+Status: implementation in progress; Stages 1–10 complete.
 
 Audience: a Luna coding agent implementing one numbered stage per prompt, in
 order. This plan starts at commit
@@ -413,6 +413,10 @@ per-input handoff already prepared in Stage 9.
 Validation: grouped/layered, import scheduling, version, destination, and
 resource-context tests plus the everyday check.
 
+**Completion:** Implemented and verified with focused grouped/layered,
+import-scheduling, version, destination, and resource-context tests plus
+`./scripts/check.sh`.
+
 ## Stage 11 — Compose Strict Collections With Resolved Identity
 
 Scope: the policy-aware strict composition core. Nested pending-item scope

internal/engine/imports.go

@@ -285,6 +285,10 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
 // the already acquired documents and graph; source handles are closed before
 // this handoff.
 func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) {
+	return composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy)
+}
+
+func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) {
 	if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 {
 		return input.composition, nil
 	}
@@ -370,7 +374,30 @@ func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy)
 		composition.importGraph = acquired.importGraph.clone()
 		return composition, nil
 	}
-	if diagnostics := traversal.composePendingItems(&scope); len(diagnostics) != 0 {
+	return composeAcquiredScopeWithPolicy(
+		&scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting,
+	)
+}
+
+// composeAcquiredScopeWithPolicy is shared by per-input and underlay-group
+// handoffs. Lower policy is carried separately from registered application
+// constraints; current strict composition continues to use its existing rules.
+func composeAcquiredScopeWithPolicy(
+	scope *acquisitionScope,
+	lower collectionPolicyScope,
+	policy sequencePolicy,
+	graph importInclusionGraph,
+	sources []SourceInfo,
+	rule *VersionRule,
+	versionState *sourceVersionState,
+	accounting *importAccounting,
+) (*composedDocument, []Diagnostic) {
+	_ = lower
+	traversal := importTraversal{
+		accounting: accounting, limits: defaultImportLimits(), rule: rule,
+		versionState: versionState, importGraph: graph.clone(),
+	}
+	if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 {
 		return nil, diagnostics
 	}
 	documents := make([]*document.Node, 0, len(scope.contributions))
@@ -381,15 +408,15 @@ func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy)
 	if len(diagnostics) != 0 {
 		return nil, diagnostics
 	}
-	if diagnostic := importedVersionClaim(documents, scope, composition, rule); diagnostic != nil {
+	if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil {
 		return nil, []Diagnostic{*diagnostic}
 	}
-	projectNestedEvidence(composition, scope)
+	projectNestedEvidence(composition, *scope)
-	projectAcquiredDestinations(composition, scope, input.sources)
+	projectAcquiredDestinations(composition, *scope, sources)
-	composition.acquisition = []acquisitionScope{cloneAcquisitionScope(scope)}
+	composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)}
 	composition.inputBytes = accounting.bytes
 	composition.inputNodes = accounting.nodes
-	composition.importGraph = acquired.importGraph.clone()
+	composition.importGraph = graph.clone()
 	return composition, nil
 }
 

internal/engine/load_request.go

@@ -37,6 +37,16 @@ type loadInput struct {
 
 type loadGroup struct{ inputs []loadInput }
 
+type acquiredUnderlayGroup struct {
+	index       int
+	scopes      []acquisitionScope
+	importGraph importInclusionGraph
+	sources     []SourceInfo
+	contexts    map[string]ResourceContext
+	inputBytes  int64
+	inputNodes  int
+}
+
 func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
 	request.ordinary = false
 	var diagnostics []Diagnostic
@@ -84,9 +94,14 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
 func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
 	groups := make([]*composedDocument, 0, len(request.underlays))
 	for index, group := range request.underlays {
-		composition, sources, contexts, diagnostics := acquireUnderlayGroupWithPolicyVersion(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy)
+		acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy)
-		input.sources = append(input.sources, sources...)
+		mergeResourceContexts(&input.contexts, acquired.contexts)
-		mergeResourceContexts(&input.contexts, contexts)
+		if len(diagnostics) != 0 {
+			input.sources = append(input.sources, acquired.sources...)
+			return input, diagnostics
+		}
+		composition, diagnostics := composeUnderlayGroup(acquired, request.protectionVersion(), request.sequencePolicy, collectionPolicyScope{})
+		input.sources = append(input.sources, acquired.sources...)
 		if len(diagnostics) != 0 {
 			return input, diagnostics
 		}
@@ -162,6 +177,15 @@ func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRu
 }
 
 func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
+	acquired, diagnostics := acquireUnderlayGroupScopes(group, index, rule, accounting, versionState, policy)
+	if len(diagnostics) != 0 {
+		return nil, acquired.sources, acquired.contexts, diagnostics
+	}
+	composition, diagnostics := composeUnderlayGroup(acquired, rule, policy, collectionPolicyScope{})
+	return composition, acquired.sources, acquired.contexts, diagnostics
+}
+
+func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*acquiredUnderlayGroup, []Diagnostic) {
 	initialBytes, initialNodes := accounting.bytes, accounting.nodes
 	visited := make(map[resourceIdentity]importRoute)
 	hostToken := &namespaceToken{}
@@ -173,10 +197,8 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
 			}
 		}
 	}
-	var scopes []acquisitionScope
+	acquiredGroup := &acquiredUnderlayGroup{index: index}
 	inclusionGraph := newImportInclusionGraph()
-	var sources []SourceInfo
-	var contexts map[string]ResourceContext
 	for inputIndex, contribution := range group.inputs {
 		options := Options[struct{}]{
 			Imports: contribution.imports, Filesystem: contribution.filesystem,
@@ -193,11 +215,11 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
 		for _, source := range acquired.sources {
 			source.Layer = LayerUnderlay
 			source.UnderlayIndex = index
-			sources = append(sources, source)
+			acquiredGroup.sources = append(acquiredGroup.sources, source)
 		}
-		mergeResourceContexts(&contexts, acquired.contexts)
+		mergeResourceContexts(&acquiredGroup.contexts, acquired.contexts)
 		if len(diagnostics) != 0 {
-			return nil, sources, contexts, projectPriorInputDiagnosticPaths(diagnostics, scopes)
+			return acquiredGroup, projectPriorInputDiagnosticPaths(diagnostics, acquiredGroup.scopes)
 		}
 		if acquired.document == nil && len(acquired.acquisition) == 0 {
 			continue
@@ -219,59 +241,66 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
 		for scopeIndex := range inputScopes {
 			reanchorAcquisitionScope(&inputScopes[scopeIndex], keyMap, anchors)
 		}
-		scopes = append(scopes, inputScopes...)
+		acquiredGroup.scopes = append(acquiredGroup.scopes, inputScopes...)
 		mergeImportInclusionGraph(&inclusionGraph, inputGraph)
 	}
-	groupScope := scheduleAcquisitionGroup(scopes, inclusionGraph)
+	acquiredGroup.importGraph = inclusionGraph.clone()
-	traversal := importTraversal{accounting: accounting, limits: defaultImportLimits(), rule: rule, importGraph: inclusionGraph}
+	acquiredGroup.inputBytes = accounting.bytes - initialBytes
-	if diagnostics := traversal.composePendingItems(&groupScope); len(diagnostics) != 0 {
+	acquiredGroup.inputNodes = accounting.nodes - initialNodes
-		return nil, sources, contexts, diagnostics
+	return acquiredGroup, nil
+}
+
+func composeUnderlayGroup(acquired *acquiredUnderlayGroup, rule *VersionRule, policy sequencePolicy, lower collectionPolicyScope) (*composedDocument, []Diagnostic) {
+	if acquired == nil {
+		return nil, nil
+	}
+	groupScope := scheduleAcquisitionGroup(acquired.scopes, acquired.importGraph)
+	if len(groupScope.contributions) == 0 {
+		return nil, nil
+	}
+	groupScope.owner = importNodeKey{namespaceKind: "group", namespaceAnchor: explicitNamespaceAnchor(acquired.index, 0), resourceKind: "scope"}
+	for contributionIndex := range groupScope.contributions {
+		groupScope.contributions[contributionIndex].scopeOwner = groupScope.owner
+	}
+	accounting := &importAccounting{
+		sources: len(acquired.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes,
+	}
+	composition, diagnostics := composeAcquiredScopeWithPolicy(
+		&groupScope, lower, policy, acquired.importGraph, acquired.sources, nil, nil, accounting,
+	)
+	if len(diagnostics) != 0 {
+		return nil, diagnostics
 	}
 	var documents []*document.Node
 	for _, contribution := range groupScope.contributions {
 		documents = append(documents, contribution.document)
 	}
-	if len(documents) == 0 {
-		return nil, sources, contexts, nil
-	}
-	composition, diagnostics := composeDocumentsWithPolicy(policy, documents...)
-	if len(diagnostics) != 0 {
-		return nil, sources, contexts, diagnostics
-	}
 	if composition != nil && rule != nil {
 		ranges := indexSequenceRanges(composition.sequenceRanges)
-		for sourceIndex, document := range documents {
+		for sourceIndex, input := range documents {
 			if groupScope.contributions[sourceIndex].role == importRoleMain {
 				continue
 			}
-			if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil {
+			if node := fragmentVersionClaimWithIndex(input, rule, sourceIndex, ranges); node != nil {
 				location := publicLocation(node.Location)
-				return nil, sources, contexts, []Diagnostic{{
+				return nil, []Diagnostic{{
 					Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(),
 					Message: "imported fragments cannot contribute to the schema-version path", Primary: &location,
 				}}
 			}
 		}
-		for sourceIndex, document := range documents {
+		for sourceIndex, input := range documents {
-			if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil {
+			if node := fragmentVersionClaimWithIndex(input, rule, sourceIndex, ranges); node != nil {
 				location := publicLocation(node.Location)
-				return nil, sources, contexts, []Diagnostic{{
+				return nil, []Diagnostic{{
 					Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(),
 					Message: "underlay inputs cannot contribute to the schema-version path", Primary: &location,
 				}}
 			}
 		}
 	}
-	composition.inputBytes = accounting.bytes - initialBytes
+	composition.inputBytes = acquired.inputBytes
-	composition.inputNodes = accounting.nodes - initialNodes
+	composition.inputNodes = acquired.inputNodes
-	groupScope.owner = importNodeKey{namespaceKind: "group", namespaceAnchor: explicitNamespaceAnchor(index, 0), resourceKind: "scope"}
-	for contributionIndex := range groupScope.contributions {
-		groupScope.contributions[contributionIndex].scopeOwner = groupScope.owner
-	}
-	composition.acquisition = []acquisitionScope{cloneAcquisitionScope(groupScope)}
-	composition.importGraph = inclusionGraph.clone()
-	projectNestedEvidence(composition, groupScope)
-	projectAcquiredDestinations(composition, groupScope, sources)
 	for pointer, origin := range composition.origins {
 		origin.Layer = LayerUnderlay
 		composition.origins[pointer] = origin
@@ -279,7 +308,7 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
 	for index := range composition.applications {
 		composition.applications[index].layer = LayerUnderlay
 	}
-	return composition, sources, contexts, nil
+	return composition, nil
 }
 
 func nativeTokenForLoadInput(input loadInput) *namespaceToken {

internal/engine/load_request_test.go

@@ -73,6 +73,50 @@ items: [{id: b}]
 	}
 }
 
+func TestUnderlayGroupAcquisitionRetainsScopesUntilComposition(t *testing.T) {
+	filesystem := NewFilesystem("handoff", fstest.MapFS{
+		"main.yml":  &fstest.MapFile{Data: []byte("configkit: {imports: [{path: child.yml}]}\nmain: retained\n")},
+		"child.yml": &fstest.MapFile{Data: []byte("child: retained\n")},
+	})
+	accounting := &importAccounting{}
+	group := loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}}
+	acquired, diagnostics := acquireUnderlayGroupScopes(group, 2, nil, accounting, nil, sequencePolicy{})
+	if len(diagnostics) != 0 {
+		t.Fatalf("acquire underlay group: %#v", diagnostics)
+	}
+	if acquired == nil || acquired.index != 2 || len(acquired.scopes) != 1 || len(acquired.scopes[0].contributions) != 2 {
+		t.Fatalf("acquisition handoff lost its source scope: %#v", acquired)
+	}
+	if len(acquired.sources) != 2 || acquired.sources[0].UnderlayIndex != 2 || acquired.sources[1].ID != "source-2" {
+		t.Fatalf("acquired source evidence = %#v", acquired.sources)
+	}
+	if acquired.inputBytes != int64(len("configkit: {imports: [{path: child.yml}]}\nmain: retained\n")+len("child: retained\n")) {
+		t.Fatalf("group input byte accounting = %d", acquired.inputBytes)
+	}
+
+	composition, diagnostics := composeUnderlayGroup(acquired, nil, sequencePolicy{}, collectionPolicyScope{})
+	if len(diagnostics) != 0 {
+		t.Fatalf("compose acquired group: %#v", diagnostics)
+	}
+	for key, want := range map[string]string{"main": "retained", "child": "retained"} {
+		value, present := documentAtPath(composition.root, Keys(key))
+		if !present || value.Text != want {
+			t.Errorf("composed %s = %#v, %v; want %q", key, value, present, want)
+		}
+	}
+	if composition.inputBytes != acquired.inputBytes || len(composition.acquisition) != 1 || len(composition.acquisition[0].contributions) != 2 {
+		t.Fatalf("composed group handoff = %#v", composition)
+	}
+
+	empty, diagnostics := acquireUnderlayGroupScopes(loadGroup{}, 3, nil, accounting, nil, sequencePolicy{})
+	if len(diagnostics) != 0 || len(empty.scopes) != 0 {
+		t.Fatalf("empty group acquisition = %#v, %#v", empty, diagnostics)
+	}
+	if composition, diagnostics := composeUnderlayGroup(empty, nil, sequencePolicy{}, collectionPolicyScope{}); len(diagnostics) != 0 || composition != nil {
+		t.Fatalf("empty group composition = %#v, %#v", composition, diagnostics)
+	}
+}
+
 func TestUnderlayNamespaceAnchorsUseInputPositionsAndFilesystemIdentity(t *testing.T) {
 	firstNamespace := NewFilesystem("same-label", fstest.MapFS{
 		"root.yml":  &fstest.MapFile{Data: []byte("items: [{id: first}]\n")},