From 628fc53d2cc899ec600fd057f2cf39eaaea8ecd1 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 23 Oct 2026 18:22:37 +0000 Subject: [PATCH] Separate underlay acquisition from composition --- docs/roadmap/implementation.md | 6 +- internal/engine/imports.go | 39 ++++++++-- internal/engine/load_request.go | 105 +++++++++++++++++---------- internal/engine/load_request_test.go | 44 +++++++++++ 4 files changed, 149 insertions(+), 45 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e0e22b6..c01713e 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Sequence Identity And Combination Implementation Plan -Status: implementation in progress; Stages 1–9 complete. +Status: implementation in progress; Stages 1–10 complete. Audience: a Luna coding agent implementing one numbered stage per prompt, in order. This plan starts at commit @@ -413,6 +413,10 @@ per-input handoff already prepared in Stage 9. Validation: grouped/layered, import scheduling, version, destination, and resource-context tests plus the everyday check. +**Completion:** Implemented and verified with focused grouped/layered, +import-scheduling, version, destination, and resource-context tests plus +`./scripts/check.sh`. + ## Stage 11 — Compose Strict Collections With Resolved Identity Scope: the policy-aware strict composition core. Nested pending-item scope diff --git a/internal/engine/imports.go b/internal/engine/imports.go index ee7ef16..738430d 100644 --- a/internal/engine/imports.go +++ b/internal/engine/imports.go @@ -285,6 +285,10 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem // the already acquired documents and graph; source handles are closed before // this handoff. func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) { + return composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy) +} + +func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) { if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 { return input.composition, nil } @@ -370,7 +374,30 @@ func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) composition.importGraph = acquired.importGraph.clone() return composition, nil } - if diagnostics := traversal.composePendingItems(&scope); len(diagnostics) != 0 { + return composeAcquiredScopeWithPolicy( + &scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting, + ) +} + +// composeAcquiredScopeWithPolicy is shared by per-input and underlay-group +// handoffs. Lower policy is carried separately from registered application +// constraints; current strict composition continues to use its existing rules. +func composeAcquiredScopeWithPolicy( + scope *acquisitionScope, + lower collectionPolicyScope, + policy sequencePolicy, + graph importInclusionGraph, + sources []SourceInfo, + rule *VersionRule, + versionState *sourceVersionState, + accounting *importAccounting, +) (*composedDocument, []Diagnostic) { + _ = lower + traversal := importTraversal{ + accounting: accounting, limits: defaultImportLimits(), rule: rule, + versionState: versionState, importGraph: graph.clone(), + } + if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 { return nil, diagnostics } documents := make([]*document.Node, 0, len(scope.contributions)) @@ -381,15 +408,15 @@ func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) if len(diagnostics) != 0 { return nil, diagnostics } - if diagnostic := importedVersionClaim(documents, scope, composition, rule); diagnostic != nil { + if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil { return nil, []Diagnostic{*diagnostic} } - projectNestedEvidence(composition, scope) - projectAcquiredDestinations(composition, scope, input.sources) - composition.acquisition = []acquisitionScope{cloneAcquisitionScope(scope)} + projectNestedEvidence(composition, *scope) + projectAcquiredDestinations(composition, *scope, sources) + composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)} composition.inputBytes = accounting.bytes composition.inputNodes = accounting.nodes - composition.importGraph = acquired.importGraph.clone() + composition.importGraph = graph.clone() return composition, nil } diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index 0288af7..a0ae32b 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -37,6 +37,16 @@ type loadInput struct { type loadGroup struct{ inputs []loadInput } +type acquiredUnderlayGroup struct { + index int + scopes []acquisitionScope + importGraph importInclusionGraph + sources []SourceInfo + contexts map[string]ResourceContext + inputBytes int64 + inputNodes int +} + func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) { request.ordinary = false var diagnostics []Diagnostic @@ -84,9 +94,14 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) { groups := make([]*composedDocument, 0, len(request.underlays)) for index, group := range request.underlays { - composition, sources, contexts, diagnostics := acquireUnderlayGroupWithPolicyVersion(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy) - input.sources = append(input.sources, sources...) - mergeResourceContexts(&input.contexts, contexts) + acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy) + mergeResourceContexts(&input.contexts, acquired.contexts) + if len(diagnostics) != 0 { + input.sources = append(input.sources, acquired.sources...) + return input, diagnostics + } + composition, diagnostics := composeUnderlayGroup(acquired, request.protectionVersion(), request.sequencePolicy, collectionPolicyScope{}) + input.sources = append(input.sources, acquired.sources...) if len(diagnostics) != 0 { return input, diagnostics } @@ -162,6 +177,15 @@ func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRu } func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) { + acquired, diagnostics := acquireUnderlayGroupScopes(group, index, rule, accounting, versionState, policy) + if len(diagnostics) != 0 { + return nil, acquired.sources, acquired.contexts, diagnostics + } + composition, diagnostics := composeUnderlayGroup(acquired, rule, policy, collectionPolicyScope{}) + return composition, acquired.sources, acquired.contexts, diagnostics +} + +func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*acquiredUnderlayGroup, []Diagnostic) { initialBytes, initialNodes := accounting.bytes, accounting.nodes visited := make(map[resourceIdentity]importRoute) hostToken := &namespaceToken{} @@ -173,10 +197,8 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver } } } - var scopes []acquisitionScope + acquiredGroup := &acquiredUnderlayGroup{index: index} inclusionGraph := newImportInclusionGraph() - var sources []SourceInfo - var contexts map[string]ResourceContext for inputIndex, contribution := range group.inputs { options := Options[struct{}]{ Imports: contribution.imports, Filesystem: contribution.filesystem, @@ -193,11 +215,11 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver for _, source := range acquired.sources { source.Layer = LayerUnderlay source.UnderlayIndex = index - sources = append(sources, source) + acquiredGroup.sources = append(acquiredGroup.sources, source) } - mergeResourceContexts(&contexts, acquired.contexts) + mergeResourceContexts(&acquiredGroup.contexts, acquired.contexts) if len(diagnostics) != 0 { - return nil, sources, contexts, projectPriorInputDiagnosticPaths(diagnostics, scopes) + return acquiredGroup, projectPriorInputDiagnosticPaths(diagnostics, acquiredGroup.scopes) } if acquired.document == nil && len(acquired.acquisition) == 0 { continue @@ -219,59 +241,66 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver for scopeIndex := range inputScopes { reanchorAcquisitionScope(&inputScopes[scopeIndex], keyMap, anchors) } - scopes = append(scopes, inputScopes...) + acquiredGroup.scopes = append(acquiredGroup.scopes, inputScopes...) mergeImportInclusionGraph(&inclusionGraph, inputGraph) } - groupScope := scheduleAcquisitionGroup(scopes, inclusionGraph) - traversal := importTraversal{accounting: accounting, limits: defaultImportLimits(), rule: rule, importGraph: inclusionGraph} - if diagnostics := traversal.composePendingItems(&groupScope); len(diagnostics) != 0 { - return nil, sources, contexts, diagnostics + acquiredGroup.importGraph = inclusionGraph.clone() + acquiredGroup.inputBytes = accounting.bytes - initialBytes + acquiredGroup.inputNodes = accounting.nodes - initialNodes + return acquiredGroup, nil +} + +func composeUnderlayGroup(acquired *acquiredUnderlayGroup, rule *VersionRule, policy sequencePolicy, lower collectionPolicyScope) (*composedDocument, []Diagnostic) { + if acquired == nil { + return nil, nil + } + groupScope := scheduleAcquisitionGroup(acquired.scopes, acquired.importGraph) + if len(groupScope.contributions) == 0 { + return nil, nil + } + groupScope.owner = importNodeKey{namespaceKind: "group", namespaceAnchor: explicitNamespaceAnchor(acquired.index, 0), resourceKind: "scope"} + for contributionIndex := range groupScope.contributions { + groupScope.contributions[contributionIndex].scopeOwner = groupScope.owner + } + accounting := &importAccounting{ + sources: len(acquired.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes, + } + composition, diagnostics := composeAcquiredScopeWithPolicy( + &groupScope, lower, policy, acquired.importGraph, acquired.sources, nil, nil, accounting, + ) + if len(diagnostics) != 0 { + return nil, diagnostics } var documents []*document.Node for _, contribution := range groupScope.contributions { documents = append(documents, contribution.document) } - if len(documents) == 0 { - return nil, sources, contexts, nil - } - composition, diagnostics := composeDocumentsWithPolicy(policy, documents...) - if len(diagnostics) != 0 { - return nil, sources, contexts, diagnostics - } if composition != nil && rule != nil { ranges := indexSequenceRanges(composition.sequenceRanges) - for sourceIndex, document := range documents { + for sourceIndex, input := range documents { if groupScope.contributions[sourceIndex].role == importRoleMain { continue } - if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil { + if node := fragmentVersionClaimWithIndex(input, rule, sourceIndex, ranges); node != nil { location := publicLocation(node.Location) - return nil, sources, contexts, []Diagnostic{{ + return nil, []Diagnostic{{ Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(), Message: "imported fragments cannot contribute to the schema-version path", Primary: &location, }} } } - for sourceIndex, document := range documents { - if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil { + for sourceIndex, input := range documents { + if node := fragmentVersionClaimWithIndex(input, rule, sourceIndex, ranges); node != nil { location := publicLocation(node.Location) - return nil, sources, contexts, []Diagnostic{{ + return nil, []Diagnostic{{ Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(), Message: "underlay inputs cannot contribute to the schema-version path", Primary: &location, }} } } } - composition.inputBytes = accounting.bytes - initialBytes - composition.inputNodes = accounting.nodes - initialNodes - groupScope.owner = importNodeKey{namespaceKind: "group", namespaceAnchor: explicitNamespaceAnchor(index, 0), resourceKind: "scope"} - for contributionIndex := range groupScope.contributions { - groupScope.contributions[contributionIndex].scopeOwner = groupScope.owner - } - composition.acquisition = []acquisitionScope{cloneAcquisitionScope(groupScope)} - composition.importGraph = inclusionGraph.clone() - projectNestedEvidence(composition, groupScope) - projectAcquiredDestinations(composition, groupScope, sources) + composition.inputBytes = acquired.inputBytes + composition.inputNodes = acquired.inputNodes for pointer, origin := range composition.origins { origin.Layer = LayerUnderlay composition.origins[pointer] = origin @@ -279,7 +308,7 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver for index := range composition.applications { composition.applications[index].layer = LayerUnderlay } - return composition, sources, contexts, nil + return composition, nil } func nativeTokenForLoadInput(input loadInput) *namespaceToken { diff --git a/internal/engine/load_request_test.go b/internal/engine/load_request_test.go index 4531453..8cfbe0b 100644 --- a/internal/engine/load_request_test.go +++ b/internal/engine/load_request_test.go @@ -73,6 +73,50 @@ items: [{id: b}] } } +func TestUnderlayGroupAcquisitionRetainsScopesUntilComposition(t *testing.T) { + filesystem := NewFilesystem("handoff", fstest.MapFS{ + "main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: child.yml}]}\nmain: retained\n")}, + "child.yml": &fstest.MapFile{Data: []byte("child: retained\n")}, + }) + accounting := &importAccounting{} + group := loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}} + acquired, diagnostics := acquireUnderlayGroupScopes(group, 2, nil, accounting, nil, sequencePolicy{}) + if len(diagnostics) != 0 { + t.Fatalf("acquire underlay group: %#v", diagnostics) + } + if acquired == nil || acquired.index != 2 || len(acquired.scopes) != 1 || len(acquired.scopes[0].contributions) != 2 { + t.Fatalf("acquisition handoff lost its source scope: %#v", acquired) + } + if len(acquired.sources) != 2 || acquired.sources[0].UnderlayIndex != 2 || acquired.sources[1].ID != "source-2" { + t.Fatalf("acquired source evidence = %#v", acquired.sources) + } + if acquired.inputBytes != int64(len("configkit: {imports: [{path: child.yml}]}\nmain: retained\n")+len("child: retained\n")) { + t.Fatalf("group input byte accounting = %d", acquired.inputBytes) + } + + composition, diagnostics := composeUnderlayGroup(acquired, nil, sequencePolicy{}, collectionPolicyScope{}) + if len(diagnostics) != 0 { + t.Fatalf("compose acquired group: %#v", diagnostics) + } + for key, want := range map[string]string{"main": "retained", "child": "retained"} { + value, present := documentAtPath(composition.root, Keys(key)) + if !present || value.Text != want { + t.Errorf("composed %s = %#v, %v; want %q", key, value, present, want) + } + } + if composition.inputBytes != acquired.inputBytes || len(composition.acquisition) != 1 || len(composition.acquisition[0].contributions) != 2 { + t.Fatalf("composed group handoff = %#v", composition) + } + + empty, diagnostics := acquireUnderlayGroupScopes(loadGroup{}, 3, nil, accounting, nil, sequencePolicy{}) + if len(diagnostics) != 0 || len(empty.scopes) != 0 { + t.Fatalf("empty group acquisition = %#v, %#v", empty, diagnostics) + } + if composition, diagnostics := composeUnderlayGroup(empty, nil, sequencePolicy{}, collectionPolicyScope{}); len(diagnostics) != 0 || composition != nil { + t.Fatalf("empty group composition = %#v, %#v", composition, diagnostics) + } +} + func TestUnderlayNamespaceAnchorsUseInputPositionsAndFilesystemIdentity(t *testing.T) { firstNamespace := NewFilesystem("same-label", fstest.MapFS{ "root.yml": &fstest.MapFile{Data: []byte("items: [{id: first}]\n")},