From 5fec16a768a33d411af835cdb99dbf88158f0503 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 23 Oct 2026 18:15:01 +0000 Subject: [PATCH] Separate input acquisition from composition --- docs/roadmap/implementation.md | 6 +- internal/engine/imports.go | 165 ++++++++++++++++++++------------ internal/engine/imports_test.go | 59 ++++++++++++ internal/engine/load.go | 17 ++-- internal/engine/load_input.go | 6 +- 5 files changed, 182 insertions(+), 71 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 7f3f81d..e0e22b6 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Sequence Identity And Combination Implementation Plan -Status: implementation in progress; Stages 1–8 complete. +Status: implementation in progress; Stages 1–9 complete. Audience: a Luna coding agent implementing one numbered stage per prompt, in order. This plan starts at commit @@ -387,6 +387,10 @@ is the next stage. Validation: the focused suites above and the everyday check. This refactor must not change source IDs or turn acquisition order into precedence order. +**Completion:** Implemented and verified with focused acquisition, scheduling, +placement, source-version, layered-equivalence, resource-lifetime, and context +tests plus `./scripts/check.sh`. + ## Stage 10 — Separate Underlay-Group Acquisition From Composition Scope: behavior-preserving group coordination in `load_request.go`, using the diff --git a/internal/engine/imports.go b/internal/engine/imports.go index 414b40c..ee7ef16 100644 --- a/internal/engine/imports.go +++ b/internal/engine/imports.go @@ -235,9 +235,6 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem } if len(diagnostics) == 0 { state.scheduleAcquisitionScopes(&state.scope) - if !deferAssembly { - diagnostics = state.composePendingItems(&state.scope) - } } closeErr := access.close() @@ -250,19 +247,19 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem if closeErr != nil { return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} } - if deferAssembly { - acquisition := state.scope - if mainIsItem { - localScope := state.scope - acquisition = acquisitionScope{ - owner: mainNodeKey, destination: destination.clone(), - contributions: []acquiredContribution{{ - source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey, - role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(), - processing: processing, pendingItem: &localScope, - }}, - } + acquisition := state.scope + if mainIsItem { + localScope := state.scope + acquisition = acquisitionScope{ + owner: mainNodeKey, destination: destination.clone(), + contributions: []acquiredContribution{{ + source: cloneSourceInfo(mainInfo), payload: body, identity: mainNodeKey, + role: importRoleMain, scopeOwner: mainNodeKey, destination: destination.clone(), + processing: processing, pendingItem: &localScope, + }}, } + } + if deferAssembly { if diagnostic := reservePendingItemPlacements(&acquisition, limits, accounting); diagnostic != nil { return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} } @@ -271,20 +268,62 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes, }, cloneSourceInfos(state.sources), nil } - if mainIsItem { - localScope := state.scope + acquired := &composedDocument{ + acquisition: []acquisitionScope{cloneAcquisitionScope(acquisition)}, + importGraph: state.importGraph.clone(), inputBytes: accounting.bytes, inputNodes: accounting.nodes, + } + input := assembledInput{ + parsed: main, composition: acquired, acquisition: cloneAcquisitionScopes(acquired.acquisition), + sources: state.sources, versionState: versionState, accounting: accounting, + } + composed, diagnostics := composeAcquiredInputWithPolicy(input, policy) + return composed, cloneSourceInfos(state.sources), diagnostics +} + +// composeAcquiredInputWithPolicy is the per-input adapter from retained +// acquisition scopes to the established strict-composition behavior. It uses +// the already acquired documents and graph; source handles are closed before +// this handoff. +func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) { + if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 { + return input.composition, nil + } + acquired := input.composition + if len(acquired.acquisition) != 1 { + return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "acquired input has an invalid composition scope", nil)} + } + scope := cloneAcquisitionScope(acquired.acquisition[0]) + var rule *VersionRule + if input.versionState != nil { + rule = input.versionState.effectiveRule + } + accounting := input.accounting + if accounting == nil { + accounting = &importAccounting{sources: len(input.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes} + } + traversal := importTraversal{ + accounting: accounting, + limits: defaultImportLimits(), rule: rule, versionState: input.versionState, + importGraph: acquired.importGraph.clone(), + } + if len(scope.contributions) == 1 && scope.contributions[0].role == importRoleMain && scope.contributions[0].pendingItem != nil { + main := scope.contributions[0] + localScope := cloneAcquisitionScope(*main.pendingItem) + if diagnostics := traversal.composePendingItems(&localScope); len(diagnostics) != 0 { + return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0)) + } documents := make([]*document.Node, 0, len(localScope.contributions)) for _, contribution := range localScope.contributions { documents = append(documents, contribution.document) } - completed, failures := composeDocuments(documents...) - if len(failures) != 0 { - return nil, cloneSourceInfos(state.sources), prependCompositionDiagnosticPaths(failures, destination.Index(0)) + completed, diagnostics := composeDocuments(documents...) + if len(diagnostics) != 0 { + return nil, prependCompositionDiagnosticPaths(diagnostics, main.destination.Index(0)) } - if versionState != nil { - protectionRule := versionState.localRule - if versionState.mode == sourceVersionModeDefault && protectionRule != nil { - prefix := destination.Index(0) + if input.versionState != nil { + protectionRule := input.versionState.localRule + if input.versionState.mode == sourceVersionModeDefault && protectionRule != nil { + prefix := main.destination.Index(0) relative, inside := relativeHistoryPath(prefix, protectionRule.Path) if !inside || !matchesPathShape(Path{segments: protectionRule.Path.segments[:len(prefix.segments)]}, prefix) { protectionRule = nil @@ -298,56 +337,60 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem if rule != nil { diagnostic.Path = rule.Path.clone() } - return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} + return nil, []Diagnostic{*diagnostic} } } completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)} projectNestedEvidence(completed, localScope) - _, placed, err := placeDocumentWithMode(completed.root, destination, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes) + _, placed, err := placeDocumentWithMode(completed.root, main.destination, placementModeSequenceItem, main.payload.Location, maxDocumentDepth, maxDocumentNodes) if err != nil { - return nil, cloneSourceInfos(state.sources), []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} + return nil, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} } - placementNodes := destinationWrapperNodeCountForMode(destination, placementModeSequenceItem) - if accounting.nodes+placementNodes > limits.nodes { - return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} + if !main.placementCharged { + placementNodes := destinationWrapperNodeCountForMode(main.destination, placementModeSequenceItem) + if accounting.nodes+placementNodes > traversal.limits.nodes { + return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", main.source.ImportedFrom)} + } + accounting.nodes += placementNodes } - accounting.nodes += placementNodes - composed, failures := composeDocumentsWithPolicy(policy, placed) - if composed != nil { - synthetic := acquisitionScope{contributions: []acquiredContribution{{ - source: cloneSourceInfo(mainInfo), payload: completed.root, document: placed, - identity: mainNodeKey, role: importRoleMain, scopeOwner: mainNodeKey, - destination: destination.clone(), processing: processing, nested: completed, - }}} - synthetic.owner = mainNodeKey - projectNestedEvidence(composed, synthetic) - projectAcquiredDestinations(composed, synthetic, state.sources) - composed.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)} - composed.inputBytes = accounting.bytes - composed.inputNodes = accounting.nodes - composed.importGraph = state.importGraph.clone() + composition, diagnostics := composeDocumentsWithPolicy(policy, placed) + if len(diagnostics) != 0 { + return nil, diagnostics } - return composed, cloneSourceInfos(state.sources), failures + synthetic := acquisitionScope{owner: main.scopeOwner, contributions: []acquiredContribution{{ + source: cloneSourceInfo(main.source), payload: completed.root, document: placed, + identity: main.identity, role: importRoleMain, scopeOwner: main.scopeOwner, + destination: main.destination.clone(), processing: main.processing, nested: completed, + }}} + projectNestedEvidence(composition, synthetic) + projectAcquiredDestinations(composition, synthetic, input.sources) + composition.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)} + composition.inputBytes = accounting.bytes + composition.inputNodes = accounting.nodes + composition.importGraph = acquired.importGraph.clone() + return composition, nil } - documents := make([]*document.Node, 0, len(state.scope.contributions)) - for _, contribution := range state.scope.contributions { + if diagnostics := traversal.composePendingItems(&scope); len(diagnostics) != 0 { + return nil, diagnostics + } + documents := make([]*document.Node, 0, len(scope.contributions)) + for _, contribution := range scope.contributions { documents = append(documents, contribution.document) } - composed, diagnostics := composeDocumentsWithPolicy(policy, documents...) - if composed != nil { - if diagnostic := importedVersionClaim(documents, state.scope, composed, rule); diagnostic != nil { - return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic} - } + composition, diagnostics := composeDocumentsWithPolicy(policy, documents...) + if len(diagnostics) != 0 { + return nil, diagnostics } - if composed != nil { - projectNestedEvidence(composed, state.scope) - projectAcquiredDestinations(composed, state.scope, state.sources) - composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)} - composed.inputBytes = accounting.bytes - composed.inputNodes = accounting.nodes - composed.importGraph = state.importGraph.clone() + if diagnostic := importedVersionClaim(documents, scope, composition, rule); diagnostic != nil { + return nil, []Diagnostic{*diagnostic} } - return composed, cloneSourceInfos(state.sources), diagnostics + projectNestedEvidence(composition, scope) + projectAcquiredDestinations(composition, scope, input.sources) + composition.acquisition = []acquisitionScope{cloneAcquisitionScope(scope)} + composition.inputBytes = accounting.bytes + composition.inputNodes = accounting.nodes + composition.importGraph = acquired.importGraph.clone() + return composition, nil } func reservePendingItemPlacements(scope *acquisitionScope, limits importLimits, accounting *importAccounting) *Diagnostic { diff --git a/internal/engine/imports_test.go b/internal/engine/imports_test.go index 378b99d..a54709e 100644 --- a/internal/engine/imports_test.go +++ b/internal/engine/imports_test.go @@ -601,6 +601,65 @@ func TestProjectNestedEvidenceUsesMappedSequenceItemIndex(t *testing.T) { } } +func TestDeferredInputCompositionUsesRetainedAcquisitionScopes(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - path: item.yml + destination: [records] + placement: sequence-item +name: main +`) + itemPath := writeImportFile(t, directory, "item.yml", `configkit: + imports: + - path: helper.yml +id: item +`) + helperPath := writeImportFile(t, directory, "helper.yml", "value: acquired\n") + accounting := &importAccounting{} + input, diagnostics := assembleBaseInputTracked(File(mainPath), Options[struct{}]{ + Imports: true, Filesystem: &FilesystemContext{Root: directory}, deferAssembly: true, + }, Path{}, accounting, nil, nil) + if len(diagnostics) != 0 { + t.Fatalf("acquire input: %#v", diagnostics) + } + if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) != 1 { + t.Fatalf("acquisition handoff was composed early: %#v", input.composition) + } + if len(input.sources) != 3 || input.sources[0].ID != "source-1" || input.sources[1].ID != "source-2" || input.sources[2].ID != "source-3" { + t.Fatalf("acquired source order/identity = %#v", input.sources) + } + if _, err := os.Stat(itemPath); err != nil { + t.Fatalf("item source disappeared before composition: %v", err) + } + if _, err := os.Stat(helperPath); err != nil { + t.Fatalf("helper source disappeared before composition: %v", err) + } + + composition, diagnostics := compositionForInputWithPolicy(input, sequencePolicy{}) + if len(diagnostics) != 0 { + t.Fatalf("compose acquired input: %#v", diagnostics) + } + if value, present := documentAtPath(composition.root, Keys("name")); !present || value.Text != "main" { + t.Fatalf("main value = %#v, %v", value, present) + } + for path, want := range map[string]string{ + "/records/0/id": "item", + "/records/0/value": "acquired", + } { + value, present := documentAtPath(composition.root, mustParsePath(t, path)) + if !present || value.Text != want { + t.Errorf("composed %s = %#v, %v; want %q", path, value, present, want) + } + } + if got, want := input.sources[1].Destination.String(), "/records"; got != want { + t.Errorf("item destination = %q, want %q", got, want) + } + if got, want := input.sources[2].Destination.String(), "/records/0"; got != want { + t.Errorf("helper destination = %q, want %q", got, want) + } +} + func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T) { t.Parallel() type config struct { diff --git a/internal/engine/load.go b/internal/engine/load.go index c74c03c..b0c3cf0 100644 --- a/internal/engine/load.go +++ b/internal/engine/load.go @@ -168,6 +168,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo mainOptions := Options[T]{Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem, destination: request.main.destination.clone(), placementMode: request.main.placementMode} mainOptions.importDefaults = snapshotImportDefaults(request.importDefaults) mainOptions.sequencePolicy = request.sequencePolicy + mainOptions.deferAssembly = request.ordinary mainOptions.SourceVersionMode = request.sourceVersionMode mainOptions.Definition.Version = request.protectionVersion() mainOptions.Overrides = request.overrides @@ -205,14 +206,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo return nil, accumulator.loadError() } } - if request.sourceVersionMode == sourceVersionModeDefault { - versionDiagnostics := checkVersion(request.definition.Version, input.document) - accumulator.add(versionDiagnostics...) - if hasErrors(versionDiagnostics) { - return nil, accumulator.loadError() - } - } - if request.ordinary && input.document != nil { + if request.ordinary && (input.document != nil || input.composition != nil) { input.composition, diagnostics = compositionForInputWithPolicy(input, request.sequencePolicy) if len(diagnostics) != 0 { accumulator.add(diagnostics...) @@ -220,6 +214,13 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo } input.document = input.composition.root } + if request.sourceVersionMode == sourceVersionModeDefault { + versionDiagnostics := checkVersion(request.definition.Version, input.document) + accumulator.add(versionDiagnostics...) + if hasErrors(versionDiagnostics) { + return nil, accumulator.loadError() + } + } if request.ordinary { input, diagnostics = applyInputLayers(request.main.source, mainOptions, input) } else { diff --git a/internal/engine/load_input.go b/internal/engine/load_input.go index d4ec6dc..16486da 100644 --- a/internal/engine/load_input.go +++ b/internal/engine/load_input.go @@ -16,6 +16,7 @@ type assembledInput struct { sources []SourceInfo contexts map[string]ResourceContext versionState *sourceVersionState + accounting *importAccounting } // acquiredContribution keeps a source-local payload beside its placed document. @@ -106,7 +107,7 @@ func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], d parsed.source.Destination = destination.clone() parsed.source.placementMode = options.placementMode } - input := assembledInput{parsed: parsed, document: parsed.document, sources: sourceInfos(parsed.source), versionState: versionState} + input := assembledInput{parsed: parsed, document: parsed.document, sources: sourceInfos(parsed.source), versionState: versionState, accounting: accounting} if len(diagnostics) != 0 { input.sources = sourceInfos(parsed.source) // Mapping value issues already identify application content. Other early @@ -265,6 +266,9 @@ func compositionForInput(input assembledInput) (*composedDocument, []Diagnostic) func compositionForInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) { if input.composition != nil { + if input.composition.root == nil && len(input.composition.acquisition) != 0 { + return composeAcquiredInputWithPolicy(input, policy) + } return input.composition, nil } if input.document == nil {