eric/ConfigKit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Separate underlay acquisition from composition

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-23 18:22:37 +00:00
parent 5fec16a768
commit 628fc53d2c
4 changed files with 149 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/engine/imports.go
View File

@@ -285,6 +285,10 @@ func loadImportedWithAccessTrackedAndProcessingPolicyDeferred(access *filesystem
// the already acquired documents and graph; source handles are closed before
// this handoff.
func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy) (*composedDocument, []Diagnostic) {
return composeAcquiredInputWithPolicies(input, collectionPolicyScope{}, policy)
}
func composeAcquiredInputWithPolicies(input assembledInput, lower collectionPolicyScope, policy sequencePolicy) (*composedDocument, []Diagnostic) {
if input.composition == nil || input.composition.root != nil || len(input.composition.acquisition) == 0 {
return input.composition, nil
}
@@ -370,7 +374,30 @@ func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy)
composition.importGraph = acquired.importGraph.clone()
return composition, nil
}
if diagnostics := traversal.composePendingItems(&scope); len(diagnostics) != 0 {
return composeAcquiredScopeWithPolicy(
&scope, lower, policy, acquired.importGraph, input.sources, rule, input.versionState, accounting,
)
}
// composeAcquiredScopeWithPolicy is shared by per-input and underlay-group
// handoffs. Lower policy is carried separately from registered application
// constraints; current strict composition continues to use its existing rules.
func composeAcquiredScopeWithPolicy(
scope *acquisitionScope,
lower collectionPolicyScope,
policy sequencePolicy,
graph importInclusionGraph,
sources []SourceInfo,
rule *VersionRule,
versionState *sourceVersionState,
accounting *importAccounting,
) (*composedDocument, []Diagnostic) {
_ = lower
traversal := importTraversal{
accounting: accounting, limits: defaultImportLimits(), rule: rule,
versionState: versionState, importGraph: graph.clone(),
}
if diagnostics := traversal.composePendingItems(scope); len(diagnostics) != 0 {
return nil, diagnostics
}
documents := make([]*document.Node, 0, len(scope.contributions))
@@ -381,15 +408,15 @@ func composeAcquiredInputWithPolicy(input assembledInput, policy sequencePolicy)
if len(diagnostics) != 0 {
return nil, diagnostics
}
if diagnostic := importedVersionClaim(documents, scope, composition, rule); diagnostic != nil {
if diagnostic := importedVersionClaim(documents, *scope, composition, rule); diagnostic != nil {
return nil, []Diagnostic{*diagnostic}
}
projectNestedEvidence(composition, scope)
projectAcquiredDestinations(composition, scope, input.sources)
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(scope)}
projectNestedEvidence(composition, *scope)
projectAcquiredDestinations(composition, *scope, sources)
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(*scope)}
composition.inputBytes = accounting.bytes
composition.inputNodes = accounting.nodes
composition.importGraph = acquired.importGraph.clone()
composition.importGraph = graph.clone()
return composition, nil
}

105
internal/engine/load_request.go
View File

@@ -37,6 +37,16 @@ type loadInput struct {
type loadGroup struct{ inputs []loadInput }
type acquiredUnderlayGroup struct {
index int
scopes []acquisitionScope
importGraph importInclusionGraph
sources []SourceInfo
contexts map[string]ResourceContext
inputBytes int64
inputNodes int
}
func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
request.ordinary = false
var diagnostics []Diagnostic
@@ -84,9 +94,14 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
groups := make([]*composedDocument, 0, len(request.underlays))
for index, group := range request.underlays {
composition, sources, contexts, diagnostics := acquireUnderlayGroupWithPolicyVersion(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy)
input.sources = append(input.sources, sources...)
mergeResourceContexts(&input.contexts, contexts)
acquired, diagnostics := acquireUnderlayGroupScopes(group, index+1, request.protectionVersion(), accounting, input.versionState, request.sequencePolicy)
mergeResourceContexts(&input.contexts, acquired.contexts)
if len(diagnostics) != 0 {
input.sources = append(input.sources, acquired.sources...)
return input, diagnostics
}
composition, diagnostics := composeUnderlayGroup(acquired, request.protectionVersion(), request.sequencePolicy, collectionPolicyScope{})
input.sources = append(input.sources, acquired.sources...)
if len(diagnostics) != 0 {
return input, diagnostics
}
@@ -162,6 +177,15 @@ func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRu
}
func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
acquired, diagnostics := acquireUnderlayGroupScopes(group, index, rule, accounting, versionState, policy)
if len(diagnostics) != 0 {
return nil, acquired.sources, acquired.contexts, diagnostics
}
composition, diagnostics := composeUnderlayGroup(acquired, rule, policy, collectionPolicyScope{})
return composition, acquired.sources, acquired.contexts, diagnostics
}
func acquireUnderlayGroupScopes(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState, policy sequencePolicy) (*acquiredUnderlayGroup, []Diagnostic) {
initialBytes, initialNodes := accounting.bytes, accounting.nodes
visited := make(map[resourceIdentity]importRoute)
hostToken := &namespaceToken{}
@@ -173,10 +197,8 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
}
}
}
var scopes []acquisitionScope
acquiredGroup := &acquiredUnderlayGroup{index: index}
inclusionGraph := newImportInclusionGraph()
var sources []SourceInfo
var contexts map[string]ResourceContext
for inputIndex, contribution := range group.inputs {
options := Options[struct{}]{
Imports: contribution.imports, Filesystem: contribution.filesystem,
@@ -193,11 +215,11 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
for _, source := range acquired.sources {
source.Layer = LayerUnderlay
source.UnderlayIndex = index
sources = append(sources, source)
acquiredGroup.sources = append(acquiredGroup.sources, source)
}
mergeResourceContexts(&contexts, acquired.contexts)
mergeResourceContexts(&acquiredGroup.contexts, acquired.contexts)
if len(diagnostics) != 0 {
return nil, sources, contexts, projectPriorInputDiagnosticPaths(diagnostics, scopes)
return acquiredGroup, projectPriorInputDiagnosticPaths(diagnostics, acquiredGroup.scopes)
}
if acquired.document == nil && len(acquired.acquisition) == 0 {
continue
@@ -219,59 +241,66 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
for scopeIndex := range inputScopes {
reanchorAcquisitionScope(&inputScopes[scopeIndex], keyMap, anchors)
}
scopes = append(scopes, inputScopes...)
acquiredGroup.scopes = append(acquiredGroup.scopes, inputScopes...)
mergeImportInclusionGraph(&inclusionGraph, inputGraph)
}
groupScope := scheduleAcquisitionGroup(scopes, inclusionGraph)
traversal := importTraversal{accounting: accounting, limits: defaultImportLimits(), rule: rule, importGraph: inclusionGraph}
if diagnostics := traversal.composePendingItems(&groupScope); len(diagnostics) != 0 {
return nil, sources, contexts, diagnostics
acquiredGroup.importGraph = inclusionGraph.clone()
acquiredGroup.inputBytes = accounting.bytes - initialBytes
acquiredGroup.inputNodes = accounting.nodes - initialNodes
return acquiredGroup, nil
}
func composeUnderlayGroup(acquired *acquiredUnderlayGroup, rule *VersionRule, policy sequencePolicy, lower collectionPolicyScope) (*composedDocument, []Diagnostic) {
if acquired == nil {
return nil, nil
}
groupScope := scheduleAcquisitionGroup(acquired.scopes, acquired.importGraph)
if len(groupScope.contributions) == 0 {
return nil, nil
}
groupScope.owner = importNodeKey{namespaceKind: "group", namespaceAnchor: explicitNamespaceAnchor(acquired.index, 0), resourceKind: "scope"}
for contributionIndex := range groupScope.contributions {
groupScope.contributions[contributionIndex].scopeOwner = groupScope.owner
}
accounting := &importAccounting{
sources: len(acquired.sources), bytes: acquired.inputBytes, nodes: acquired.inputNodes,
}
composition, diagnostics := composeAcquiredScopeWithPolicy(
&groupScope, lower, policy, acquired.importGraph, acquired.sources, nil, nil, accounting,
)
if len(diagnostics) != 0 {
return nil, diagnostics
}
var documents []*document.Node
for _, contribution := range groupScope.contributions {
documents = append(documents, contribution.document)
}
if len(documents) == 0 {
return nil, sources, contexts, nil
}
composition, diagnostics := composeDocumentsWithPolicy(policy, documents...)
if len(diagnostics) != 0 {
return nil, sources, contexts, diagnostics
}
if composition != nil && rule != nil {
ranges := indexSequenceRanges(composition.sequenceRanges)
for sourceIndex, document := range documents {
for sourceIndex, input := range documents {
if groupScope.contributions[sourceIndex].role == importRoleMain {
continue
}
if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil {
if node := fragmentVersionClaimWithIndex(input, rule, sourceIndex, ranges); node != nil {
location := publicLocation(node.Location)
return nil, sources, contexts, []Diagnostic{{
return nil, []Diagnostic{{
Code: CodeImportVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "imported fragments cannot contribute to the schema-version path", Primary: &location,
}}
}
}
for sourceIndex, document := range documents {
if node := fragmentVersionClaimWithIndex(document, rule, sourceIndex, ranges); node != nil {
for sourceIndex, input := range documents {
if node := fragmentVersionClaimWithIndex(input, rule, sourceIndex, ranges); node != nil {
location := publicLocation(node.Location)
return nil, sources, contexts, []Diagnostic{{
return nil, []Diagnostic{{
Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(),
Message: "underlay inputs cannot contribute to the schema-version path", Primary: &location,
}}
}
}
}
composition.inputBytes = accounting.bytes - initialBytes
composition.inputNodes = accounting.nodes - initialNodes
groupScope.owner = importNodeKey{namespaceKind: "group", namespaceAnchor: explicitNamespaceAnchor(index, 0), resourceKind: "scope"}
for contributionIndex := range groupScope.contributions {
groupScope.contributions[contributionIndex].scopeOwner = groupScope.owner
}
composition.acquisition = []acquisitionScope{cloneAcquisitionScope(groupScope)}
composition.importGraph = inclusionGraph.clone()
projectNestedEvidence(composition, groupScope)
projectAcquiredDestinations(composition, groupScope, sources)
composition.inputBytes = acquired.inputBytes
composition.inputNodes = acquired.inputNodes
for pointer, origin := range composition.origins {
origin.Layer = LayerUnderlay
composition.origins[pointer] = origin
@@ -279,7 +308,7 @@ func acquireUnderlayGroupWithPolicyVersion(group loadGroup, index int, rule *Ver
for index := range composition.applications {
composition.applications[index].layer = LayerUnderlay
}
return composition, sources, contexts, nil
return composition, nil
}
func nativeTokenForLoadInput(input loadInput) *namespaceToken {

44
internal/engine/load_request_test.go
View File

@@ -73,6 +73,50 @@ items: [{id: b}]
}
}
func TestUnderlayGroupAcquisitionRetainsScopesUntilComposition(t *testing.T) {
filesystem := NewFilesystem("handoff", fstest.MapFS{
"main.yml": &fstest.MapFile{Data: []byte("configkit: {imports: [{path: child.yml}]}\nmain: retained\n")},
"child.yml": &fstest.MapFile{Data: []byte("child: retained\n")},
})
accounting := &importAccounting{}
group := loadGroup{inputs: []loadInput{{source: filesystem.File("main.yml"), imports: true}}}
acquired, diagnostics := acquireUnderlayGroupScopes(group, 2, nil, accounting, nil, sequencePolicy{})
if len(diagnostics) != 0 {
t.Fatalf("acquire underlay group: %#v", diagnostics)
}
if acquired == nil || acquired.index != 2 || len(acquired.scopes) != 1 || len(acquired.scopes[0].contributions) != 2 {
t.Fatalf("acquisition handoff lost its source scope: %#v", acquired)
}
if len(acquired.sources) != 2 || acquired.sources[0].UnderlayIndex != 2 || acquired.sources[1].ID != "source-2" {
t.Fatalf("acquired source evidence = %#v", acquired.sources)
}
if acquired.inputBytes != int64(len("configkit: {imports: [{path: child.yml}]}\nmain: retained\n")+len("child: retained\n")) {
t.Fatalf("group input byte accounting = %d", acquired.inputBytes)
}
composition, diagnostics := composeUnderlayGroup(acquired, nil, sequencePolicy{}, collectionPolicyScope{})
if len(diagnostics) != 0 {
t.Fatalf("compose acquired group: %#v", diagnostics)
}
for key, want := range map[string]string{"main": "retained", "child": "retained"} {
value, present := documentAtPath(composition.root, Keys(key))
if !present || value.Text != want {
t.Errorf("composed %s = %#v, %v; want %q", key, value, present, want)
}
}
if composition.inputBytes != acquired.inputBytes || len(composition.acquisition) != 1 || len(composition.acquisition[0].contributions) != 2 {
t.Fatalf("composed group handoff = %#v", composition)
}
empty, diagnostics := acquireUnderlayGroupScopes(loadGroup{}, 3, nil, accounting, nil, sequencePolicy{})
if len(diagnostics) != 0 || len(empty.scopes) != 0 {
t.Fatalf("empty group acquisition = %#v, %#v", empty, diagnostics)
}
if composition, diagnostics := composeUnderlayGroup(empty, nil, sequencePolicy{}, collectionPolicyScope{}); len(diagnostics) != 0 || composition != nil {
t.Fatalf("empty group composition = %#v, %#v", composition, diagnostics)
}
}
func TestUnderlayNamespaceAnchorsUseInputPositionsAndFilesystemIdentity(t *testing.T) {
firstNamespace := NewFilesystem("same-label", fstest.MapFS{
"root.yml": &fstest.MapFile{Data: []byte("items: [{id: first}]\n")},