Separate input acquisition from composition

internal/engine/imports.go

@@ -235,9 +235,6 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
 	}
 	if len(diagnostics) == 0 {
 		state.scheduleAcquisitionScopes(&state.scope)
-		if !deferAssembly {
-			diagnostics = state.composePendingItems(&state.scope)
-		}
 	}
 
 	closeErr := access.close()
@@ -250,19 +247,19 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
 	if closeErr != nil {
 		return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)}
 	}
-	if deferAssembly {
-		acquisition := state.scope
-		if mainIsItem {
-			localScope := state.scope
-			acquisition = acquisitionScope{
-				owner: mainNodeKey, destination: destination.clone(),
-				contributions: []acquiredContribution{{
-					source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey,
-					role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(),
-					processing: processing, pendingItem: &localScope,
-				}},
-			}
+	acquisition := state.scope
+	if mainIsItem {
+		localScope := state.scope
+		acquisition = acquisitionScope{
+			owner: mainNodeKey, destination: destination.clone(),
+			contributions: []acquiredContribution{{
+				source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey,
+				role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(),
+				processing: processing, pendingItem: &localScope,
+			}},
 		}
+	}
+	if deferAssembly {
 		if diagnostic := reservePendingItemPlacements(&acquisition, limits, accounting); diagnostic != nil {
 			return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
 		}
@@ -271,20 +268,62 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
 			importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes,
 		}, cloneSourceInfos(state.sources), nil
 	}
-	if mainIsItem {
-		localScope := state.scope
+	acquired := &composedDocument{
+		acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)},
+		importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes,
+	}
+	input := assembledInput{
+		parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition),
+		sources: state.sources, versionState: versionState, accounting: accounting,
+	}
+	composed, diagnostics := composeAcquiredInputWithPolicy(input, policy)
+	return composed, cloneSourceInfos(state.sources), diagnostics
+}
+
+// composeAcquiredInputWithPolicy is the per-input adapter from retained
+// acquisition scopes to the established strict-composition behavior. It uses
+// the already acquired documents and graph; source handles are closed before
+// this handoff.
+func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) {
+	if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 {
+		return input.composition, nil
+	}
+	acquired := input.composition
+	if len(acquired.acquisition) != 1 {
+		return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "acquired input has an invalid composition scope", nil)}
+	}
+	scope := cloneAcquisitionScope(acquired.acquisition[0])
+	var rule *VersionRule
+	if input.versionState != nil {
+		rule = input.versionState.effectiveRule
+	}
+	accounting := input.accounting
+	if accounting == nil {
+		accounting = &importAccounting{sources: len(input.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes}
+	}
+	traversal := importTraversal{
+		accounting: accounting,
+		limits:     defaultImportLimits(), rule: rule, versionState: input.versionState,
+		importGraph: acquired.importGraph.clone(),
+	}
+	if len(scope.contributions) == 1 && scope.contributions[0].role == importRoleMain && scope.contributions[0].pendingItem != nil {
+		main := scope.contributions[0]
+		localScope := cloneAcquisitionScope(*main.pendingItem)
+		if diagnostics := traversal.composePendingItems(&localScope); len(diagnostics) != 0 {
+			return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0))
+		}
 		documents := make([]*document.Node, 0, len(localScope.contributions))
 		for _, contribution := range localScope.contributions {
 			documents = append(documents, contribution.document)
 		}
-		completed, failures := composeDocuments(documents...)
-		if len(failures) != 0 {
-			return nil, cloneSourceInfos(state.sources), prependCompositionDiagnosticPaths(failures, destination.Index(0))
+		completed, diagnostics := composeDocuments(documents...)
+		if len(diagnostics) != 0 {
+			return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0))
 		}
-		if versionState != nil {
-			protectionRule := versionState.localRule
-			if versionState.mode == sourceVersionModeDefault && protectionRule != nil {
-				prefix := destination.Index(0)
+		if input.versionState != nil {
+			protectionRule := input.versionState.localRule
+			if input.versionState.mode == sourceVersionModeDefault && protectionRule != nil {
+				prefix := main.destination.Index(0)
 				relative, inside := relativeHistoryPath(prefix, protectionRule.Path)
 				if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) {
 					protectionRule = nil
@@ -298,56 +337,60 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
 				if rule != nil {
 					diagnostic.Path = rule.Path.clone()
 				}
-				return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
+				return nil, []Diagnostic{*diagnostic}
 			}
 		}
 		completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)}
 		projectNestedEvidence(completed, localScope)
-		_, placed, err := placeDocumentWithMode(completed.root, destination, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes)
+		_, placed, err := placeDocumentWithMode(completed.root, main.destination, placementModeSequenceItem, main.payload.Location, maxDocumentDepth, maxDocumentNodes)
 		if err != nil {
-			return nil, cloneSourceInfos(state.sources), []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
+			return nil, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
 		}
-		placementNodes := destinationWrapperNodeCountForMode(destination, placementModeSequenceItem)
-		if accounting.nodes+placementNodes > limits.nodes {
-			return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)}
+		if !main.placementCharged {
+			placementNodes := destinationWrapperNodeCountForMode(main.destination, placementModeSequenceItem)
+			if accounting.nodes+placementNodes > traversal.limits.nodes {
+				return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", main.source.ImportedFrom)}
+			}
+			accounting.nodes += placementNodes
 		}
-		accounting.nodes += placementNodes
-		composed, failures := composeDocumentsWithPolicy(policy, placed)
-		if composed != nil {
-			synthetic := acquisitionScope{contributions: []acquiredContribution{{
-				source: cloneSourceInfo(mainInfo), payload: completed.root, document: placed,
-				identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey,
-				destination: destination.clone(), processing: processing, nested: completed,
-			}}}
-			synthetic.owner = mainNodeKey
-			projectNestedEvidence(composed, synthetic)
-			projectAcquiredDestinations(composed, synthetic, state.sources)
-			composed.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)}
-			composed.inputBytes = accounting.bytes
-			composed.inputNodes = accounting.nodes
-			composed.importGraph = state.importGraph.clone()
+		composition, diagnostics := composeDocumentsWithPolicy(policy, placed)
+		if len(diagnostics) != 0 {
+			return nil, diagnostics
 		}
-		return composed, cloneSourceInfos(state.sources), failures
+		synthetic := acquisitionScope{owner: main.scopeOwner, contributions: []acquiredContribution{{
+			source: cloneSourceInfo(main.source), payload: completed.root, document: placed,
+			identity: main.identity, role: importRoleMain, scopeOwner: main.scopeOwner,
+			destination: main.destination.clone(), processing: main.processing, nested: completed,
+		}}}
+		projectNestedEvidence(composition, synthetic)
+		projectAcquiredDestinations(composition, synthetic, input.sources)
+		composition.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)}
+		composition.inputBytes = accounting.bytes
+		composition.inputNodes = accounting.nodes
+		composition.importGraph = acquired.importGraph.clone()
+		return composition, nil
 	}
-	documents := make([]*document.Node, 0, len(state.scope.contributions))
-	for _, contribution := range state.scope.contributions {
+	if diagnostics := traversal.composePendingItems(&scope); len(diagnostics) != 0 {
+		return nil, diagnostics
+	}
+	documents := make([]*document.Node, 0, len(scope.contributions))
+	for _, contribution := range scope.contributions {
 		documents = append(documents, contribution.document)
 	}
-	composed, diagnostics := composeDocumentsWithPolicy(policy, documents...)
-	if composed != nil {
-		if diagnostic := importedVersionClaim(documents, state.scope, composed, rule); diagnostic != nil {
-			return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
-		}
+	composition, diagnostics := composeDocumentsWithPolicy(policy, documents...)
+	if len(diagnostics) != 0 {
+		return nil, diagnostics
 	}
-	if composed != nil {
-		projectNestedEvidence(composed, state.scope)
-		projectAcquiredDestinations(composed, state.scope, state.sources)
-		composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)}
-		composed.inputBytes = accounting.bytes
-		composed.inputNodes = accounting.nodes
-		composed.importGraph = state.importGraph.clone()
+	if diagnostic := importedVersionClaim(documents, scope, composition, rule); diagnostic != nil {
+		return nil, []Diagnostic{*diagnostic}
 	}
-	return composed, cloneSourceInfos(state.sources), diagnostics
+	projectNestedEvidence(composition, scope)
+	projectAcquiredDestinations(composition, scope, input.sources)
+	composition.acquisition = []acquisitionScope{cloneAcquisitionScope(scope)}
+	composition.inputBytes = accounting.bytes
+	composition.inputNodes = accounting.nodes
+	composition.importGraph = acquired.importGraph.clone()
+	return composition, nil
 }
 
 func reservePendingItemPlacements(scope *acquisitionScope, limits importLimits, accounting *importAccounting) *Diagnostic {

internal/engine/imports_test.go

@@ -601,6 +601,65 @@ func TestProjectNestedEvidenceUsesMappedSequenceItemIndex(t *testing.T) {
 	}
 }
 
+func TestDeferredInputCompositionUsesRetainedAcquisitionScopes(t *testing.T) {
+	directory := t.TempDir()
+	mainPath := writeImportFile(t, directory, "main.yml", `configkit:
+  imports:
+    - path: item.yml
+      destination: [records]
+      placement: sequence-item
+name: main
+`)
+	itemPath := writeImportFile(t, directory, "item.yml", `configkit:
+  imports:
+    - path: helper.yml
+id: item
+`)
+	helperPath := writeImportFile(t, directory, "helper.yml", "value: acquired\n")
+	accounting := &importAccounting{}
+	input, diagnostics := assembleBaseInputTracked(File(mainPath), Options[struct{}]{
+		Imports: true, Filesystem: &FilesystemContext{Root: directory}, deferAssembly: true,
+	}, Path{}, accounting, nil, nil)
+	if len(diagnostics) != 0 {
+		t.Fatalf("acquire input: %#v", diagnostics)
+	}
+	if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) != 1 {
+		t.Fatalf("acquisition handoff was composed early: %#v", input.composition)
+	}
+	if len(input.sources) != 3 || input.sources[0].ID != "source-1" || input.sources[1].ID != "source-2" || input.sources[2].ID != "source-3" {
+		t.Fatalf("acquired source order/identity = %#v", input.sources)
+	}
+	if _, err := os.Stat(itemPath); err != nil {
+		t.Fatalf("item source disappeared before composition: %v", err)
+	}
+	if _, err := os.Stat(helperPath); err != nil {
+		t.Fatalf("helper source disappeared before composition: %v", err)
+	}
+
+	composition, diagnostics := compositionForInputWithPolicy(input, sequencePolicy{})
+	if len(diagnostics) != 0 {
+		t.Fatalf("compose acquired input: %#v", diagnostics)
+	}
+	if value, present := documentAtPath(composition.root, Keys("name")); !present || value.Text != "main" {
+		t.Fatalf("main value = %#v, %v", value, present)
+	}
+	for path, want := range map[string]string{
+		"/records/0/id":    "item",
+		"/records/0/value": "acquired",
+	} {
+		value, present := documentAtPath(composition.root, mustParsePath(t, path))
+		if !present || value.Text != want {
+			t.Errorf("composed %s = %#v, %v; want %q", path, value, present, want)
+		}
+	}
+	if got, want := input.sources[1].Destination.String(), "/records"; got != want {
+		t.Errorf("item destination = %q, want %q", got, want)
+	}
+	if got, want := input.sources[2].Destination.String(), "/records/0"; got != want {
+		t.Errorf("helper destination = %q, want %q", got, want)
+	}
+}
+
 func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T) {
 	t.Parallel()
 	type config struct {

internal/engine/load.go

@@ -168,6 +168,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
 	mainOptions := Options[T]{Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(), placementMode: request.main.placementMode}
 	mainOptions.importDefaults = snapshotImportDefaults(request.importDefaults)
 	mainOptions.sequencePolicy = request.sequencePolicy
+	mainOptions.deferAssembly = request.ordinary
 	mainOptions.SourceVersionMode = request.sourceVersionMode
 	mainOptions.Definition.Version = request.protectionVersion()
 	mainOptions.Overrides = request.overrides
@@ -205,14 +206,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
 			return nil, accumulator.loadError()
 		}
 	}
-	if request.sourceVersionMode == sourceVersionModeDefault {
-		versionDiagnostics := checkVersion(request.definition.Version, input.document)
-		accumulator.add(versionDiagnostics...)
-		if hasErrors(versionDiagnostics) {
-			return nil, accumulator.loadError()
-		}
-	}
-	if request.ordinary && input.document != nil {
+	if request.ordinary && (input.document != nil || input.composition != nil) {
 		input.composition, diagnostics = compositionForInputWithPolicy(input, request.sequencePolicy)
 		if len(diagnostics) != 0 {
 			accumulator.add(diagnostics...)
@@ -220,6 +214,13 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
 		}
 		input.document = input.composition.root
 	}
+	if request.sourceVersionMode == sourceVersionModeDefault {
+		versionDiagnostics := checkVersion(request.definition.Version, input.document)
+		accumulator.add(versionDiagnostics...)
+		if hasErrors(versionDiagnostics) {
+			return nil, accumulator.loadError()
+		}
+	}
 	if request.ordinary {
 		input, diagnostics = applyInputLayers(request.main.source, mainOptions, input)
 	} else {

internal/engine/load_input.go

@@ -16,6 +16,7 @@ type assembledInput struct {
 	sources      []SourceInfo
 	contexts     map[string]ResourceContext
 	versionState *sourceVersionState
+	accounting   *importAccounting
 }
 
 // acquiredContribution keeps a source-local payload beside its placed document.
@@ -106,7 +107,7 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d
 		parsed.source.Destination = destination.clone()
 		parsed.source.placementMode = options.placementMode
 	}
-	input := assembledInput{parsed: parsed, document: parsed.document, sources: sourceInfos(parsed.source), versionState: versionState}
+	input := assembledInput{parsed: parsed, document: parsed.document, sources: sourceInfos(parsed.source), versionState: versionState, accounting: accounting}
 	if len(diagnostics) != 0 {
 		input.sources = sourceInfos(parsed.source)
 		// Mapping value issues already identify application content. Other early
@@ -265,6 +266,9 @@ func compositionForInput(input assembledInput) (*composedDocument, []Diagnostic)
 
 func compositionForInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) {
 	if input.composition != nil {
+		if input.composition.root == nil && len(input.composition.acquisition) != 0 {
+			return composeAcquiredInputWithPolicy(input, policy)
+		}
 		return input.composition, nil
 	}
 	if input.document == nil {