Assemble imported sequence-item scopes

This commit is contained in:
2026-10-16 16:20:14 +00:00
parent 1a04bb3835
commit 25cf6d32b6
9 changed files with 722 additions and 27 deletions

View File

@@ -884,6 +884,25 @@ walk failures, matched directories, invalid documents, escapes, and repeated tar
remain errors. Final candidates are bounded by the remaining source limit before remain errors. Final candidates are bounded by the remaining source limit before
their contents are accumulated. their contents are accumulated.
Each path or glob declaration may include `placement: mapping` or
`placement: sequence-item`. Omission means `mapping` for that declaration,
including every match of a glob; placement is not inherited. Sequence-item
placement requires a nonempty effective destination and wraps the imported
mapping as one element of a sequence there. It does not split source sequences
or enable sequence accumulation: overlapping sequences still follow the strict
composition rules above.
An item import first opens a local mapping scope. Its body and ordinary helper
imports compose within that record. An omitted helper destination or
`destination: []` places the helper at the record root; other destinations are
relative to the immediate importing source. A helper may supply fields such as
the record's `id`. An explicit nested `sequence-item` import creates a sequence
inside the record, and its placement is independent of the enclosing item.
At the record root, an item import must name a nonempty nested destination.
Physical paths continue to resolve from the file that declares each import.
An actual empty mapping contributes an empty item, while an optional unmatched
path or glob contributes nothing.
For host files, the configuration root defaults to the selected file's canonical For host files, the configuration root defaults to the selected file's canonical
directory. A host file may supply `FilesystemContext.Root` to widen that root but directory. A host file may supply `FilesystemContext.Root` to widen that root but
may not supply `BaseDir` or a native `FS`. Byte, reader, and mapping inputs with host context may not supply `BaseDir` or a native `FS`. Byte, reader, and mapping inputs with host context

View File

@@ -1,6 +1,6 @@
# Implementation Plan: Sequence Placement and Composition # Implementation Plan: Sequence Placement and Composition
Status: implementation in progress; stages 1–7 completed. Status: implementation in progress; stages 1–8 completed.
## Purpose and authority ## Purpose and authority
@@ -505,6 +505,20 @@ Validation: import/directive/glob/destination/composition tests and
`./scripts/check.sh`. Completion: private item imports construct the agreed `./scripts/check.sh`. Completion: private item imports construct the agreed
records and preserve physical acquisition evidence. records and preserve physical acquisition evidence.
Completion evidence: import declarations now parse per-declaration placement,
and item imports compose their body and ordinary helpers in a local mapping
scope before wrapping the completed record. Nested item scopes, glob matches,
relative and transitive helper destinations, source origins/history, and
resolved underlay destinations are retained. Configuration documentation
describes only the implemented import-placement subset; private registration
controls remain unadvertised. These checks passed with Go `go1.26.9 linux/amd64`:
- `GOTOOLCHAIN=local GOPROXY=off go test ./internal/engine -run 'Import|Directive|Glob|Destination|Compose|NativeFilesystem' -count=1`
- `GOTOOLCHAIN=local ./scripts/check.sh`
- `git diff --check`
No environmental limitation affected validation; checks ran on Linux/amd64.
## Stage 9 — Wire strict policy through the complete loading lifecycle ## Stage 9 — Wire strict policy through the complete loading lifecycle
Scope: ordinary/grouped engine loading and every strict-tier entry point. Scope: ordinary/grouped engine loading and every strict-tier entry point.

View File

@@ -45,6 +45,9 @@ func newAssemblyAccumulator(owned *composedDocument) *assemblyAccumulator {
// cloneComposedDocument detaches mutable composition and provenance state; // cloneComposedDocument detaches mutable composition and provenance state;
// history application trees are immutable and retain their existing sharing. // history application trees are immutable and retain their existing sharing.
func cloneComposedDocument(base *composedDocument) *composedDocument { func cloneComposedDocument(base *composedDocument) *composedDocument {
if base == nil {
return nil
}
return &composedDocument{ return &composedDocument{
root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims), root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims),
acquisition: cloneAcquisitionScopes(base.acquisition), acquisition: cloneAcquisitionScopes(base.acquisition),

View File

@@ -10,11 +10,14 @@ import (
const directiveKey = "configkit" const directiveKey = "configkit"
type importDeclaration struct { type importDeclaration struct {
path string path string
glob string glob string
destination Path destination Path
optional bool placement placementMode
location Location placementPath Path
placementLocation *Location
optional bool
location Location
} }
func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) { func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) {
@@ -70,7 +73,7 @@ func parseImportDeclaration(node *document.Node, declarationPath Path) (importDe
if node.Kind != document.Mapping { if node.Kind != document.Mapping {
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)} return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)}
} }
declaration := importDeclaration{location: publicLocation(node.Location)} declaration := importDeclaration{location: publicLocation(node.Location), placementPath: declarationPath.clone()}
seenTarget := false seenTarget := false
for _, entry := range node.Mapping { for _, entry := range node.Mapping {
fieldPath := declarationPath.Key(entry.Key) fieldPath := declarationPath.Key(entry.Key)
@@ -112,6 +115,21 @@ func parseImportDeclaration(node *document.Node, declarationPath Path) (importDe
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must contain only non-reserved mapping keys", entry.Value)} return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must contain only non-reserved mapping keys", entry.Value)}
} }
declaration.destination = normalized declaration.destination = normalized
case "placement":
declaration.placementPath = fieldPath.clone()
placementLocation := publicLocation(entry.Value.Location)
declaration.placementLocation = &placementLocation
if entry.Value.Kind != document.String {
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)}
}
switch entry.Value.Text {
case "mapping":
declaration.placement = placementModeMapping
case "sequence-item":
declaration.placement = placementModeSequenceItem
default:
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)}
}
default: default:
return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)} return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)}
} }

View File

@@ -51,11 +51,13 @@ type importTraversal struct {
rule *VersionRule rule *VersionRule
versionState *sourceVersionState versionState *sourceVersionState
scope acquisitionScope scope acquisitionScope
sources []SourceInfo itemScope bool
active map[resourceIdentity]int itemPrefix Path
visited map[resourceIdentity]importRoute sources []SourceInfo
stack []importRoute active map[resourceIdentity]int
visited map[resourceIdentity]importRoute
stack []importRoute
} }
func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
@@ -128,6 +130,10 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa
} }
mainInfo := cloneSourceInfo(*main.source) mainInfo := cloneSourceInfo(*main.source)
mainInfo.Destination = destination.clone() mainInfo.Destination = destination.clone()
mainIsItem := main.source.placementMode == placementModeSequenceItem
if mainIsItem {
mainInfo.destinationUnresolved = true
}
if mainPath != "" { if mainPath != "" {
mainInfo.Path = mainPath mainInfo.Path = mainPath
} }
@@ -148,6 +154,12 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa
sources: []SourceInfo{mainInfo}, sources: []SourceInfo{mainInfo},
active: make(map[resourceIdentity]int), visited: visited, active: make(map[resourceIdentity]int), visited: visited,
} }
if mainIsItem {
state.itemScope = true
state.itemPrefix = destination.clone()
state.scope.destination = Path{}
state.sources[0] = cloneSourceInfo(mainInfo)
}
if state.visited == nil { if state.visited == nil {
state.visited = make(map[resourceIdentity]importRoute) state.visited = make(map[resourceIdentity]importRoute)
} }
@@ -174,19 +186,26 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa
body, declarations, diagnostics = splitImportDirectives(main.document) body, declarations, diagnostics = splitImportDirectives(main.document)
} }
if len(diagnostics) == 0 { if len(diagnostics) == 0 {
_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) if mainIsItem {
placementNodes := destinationWrapperNodeCount(destination)
if err != nil {
diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
} else if accounting.nodes+placementNodes > limits.nodes {
diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)}
} else {
accounting.nodes += placementNodes
state.scope.contributions = append(state.scope.contributions, acquiredContribution{ state.scope.contributions = append(state.scope.contributions, acquiredContribution{
source: cloneSourceInfo(mainInfo), payload: body, document: placed, source: cloneSourceInfo(mainInfo), payload: body, document: body, processing: processing,
destination: destination.clone(), processing: processing,
}) })
diagnostics = state.walkDeclarations(basePath, declarations, 0, destination) diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{})
} else {
_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes)
placementNodes := destinationWrapperNodeCount(destination)
if err != nil {
diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
} else if accounting.nodes+placementNodes > limits.nodes {
diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)}
} else {
accounting.nodes += placementNodes
state.scope.contributions = append(state.scope.contributions, acquiredContribution{
source: cloneSourceInfo(mainInfo), payload: body, document: placed,
destination: destination.clone(), processing: processing,
})
diagnostics = state.walkDeclarations(basePath, declarations, 0, destination)
}
} }
} }
@@ -197,12 +216,49 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa
if closeErr != nil { if closeErr != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)}
} }
if mainIsItem {
localScope := state.scope
documents := make([]*document.Node, 0, len(localScope.contributions))
for _, contribution := range localScope.contributions {
documents = append(documents, contribution.document)
}
completed, failures := composeDocuments(documents...)
if len(failures) != 0 {
return nil, cloneSourceInfos(state.sources), prependDiagnosticPaths(failures, destination.Index(0))
}
completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)}
projectNestedEvidence(completed, localScope)
_, placed, err := placeDocumentWithMode(completed.root, destination, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return nil, cloneSourceInfos(state.sources), []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
}
placementNodes := destinationWrapperNodeCountForMode(destination, placementModeSequenceItem)
if accounting.nodes+placementNodes > limits.nodes {
return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)}
}
accounting.nodes += placementNodes
composed, failures := composeDocuments(placed)
if composed != nil {
synthetic := acquisitionScope{contributions: []acquiredContribution{{
source: cloneSourceInfo(mainInfo), payload: completed.root, document: placed,
destination: destination.clone(), processing: processing, nested: completed,
}}}
projectNestedEvidence(composed, synthetic)
projectAcquiredDestinations(composed, synthetic, state.sources)
composed.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)}
composed.inputBytes = accounting.bytes
composed.inputNodes = accounting.nodes
}
return composed, cloneSourceInfos(state.sources), failures
}
documents := make([]*document.Node, 0, len(state.scope.contributions)) documents := make([]*document.Node, 0, len(state.scope.contributions))
for _, contribution := range state.scope.contributions { for _, contribution := range state.scope.contributions {
documents = append(documents, contribution.document) documents = append(documents, contribution.document)
} }
composed, diagnostics := composeDocuments(documents...) composed, diagnostics := composeDocuments(documents...)
if composed != nil { if composed != nil {
projectNestedEvidence(composed, state.scope)
projectAcquiredDestinations(composed, state.scope, state.sources)
composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)} composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)}
composed.inputBytes = accounting.bytes composed.inputBytes = accounting.bytes
composed.inputNodes = accounting.nodes composed.inputNodes = accounting.nodes
@@ -260,6 +316,14 @@ func (s *importTraversal) walkDeclarations(basePath string, declarations []impor
func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, destination, relativeDestination Path, depth int) []Diagnostic { func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, destination, relativeDestination Path, depth int) []Diagnostic {
location := declaration.location location := declaration.location
if declaration.placement == placementModeSequenceItem && len(destination.segments) == 0 {
return []Diagnostic{{
Code: CodeImportInvalid, Severity: SeverityError,
Path: declaration.placementPath.clone(),
Message: "sequence-item placement requires a nonempty destination",
Primary: cloneLocation(declaration.placementLocation),
}}
}
if depth > s.limits.depth { if depth > s.limits.depth {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)}
} }
@@ -312,7 +376,11 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
kind = SourceKindFilesystem kind = SourceKindFilesystem
label = s.access.label + "/" + canonical label = s.access.label + "/" + canonical
} }
info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: destination.clone(), Layer: LayerBase, Kind: kind, ImportedFrom: &location} info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location}
if declaration.placement == placementModeSequenceItem {
info.placementMode = placementModeSequenceItem
}
info.destinationUnresolved = s.itemScope || declaration.placement == placementModeSequenceItem
if kind == SourceKindFilesystem { if kind == SourceKindFilesystem {
info.namespace = s.access.token info.namespace = s.access.token
} }
@@ -356,11 +424,26 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
return diagnostics return diagnostics
} }
} }
_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) placementDestination := destination
mode := placementModeMapping
if declaration.placement == placementModeSequenceItem {
mode = placementModeSequenceItem
}
if mode == placementModeSequenceItem {
route := importRoute{canonical: canonical, locations: currentRoute.locations}
s.active[identity] = len(s.stack)
s.visited[identity] = route
s.stack = append(s.stack, route)
diagnostics := s.visitItemSource(canonical, declaration, destination, relativeDestination, depth, info, body, declarations, processing)
delete(s.active, identity)
s.stack = s.stack[:len(s.stack)-1]
return diagnostics
}
_, placed, err := placeDocument(body, placementDestination, body.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil { if err != nil {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)} return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)}
} }
placementNodes := destinationWrapperNodeCount(destination) placementNodes := destinationWrapperNodeCount(placementDestination)
if s.accounting.nodes+placementNodes > s.limits.nodes { if s.accounting.nodes+placementNodes > s.limits.nodes {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
} }
@@ -380,7 +463,7 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
route := importRoute{canonical: canonical, locations: currentRoute.locations} route := importRoute{canonical: canonical, locations: currentRoute.locations}
s.scope.contributions = append(s.scope.contributions, acquiredContribution{ s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(info), payload: body, document: placed, source: cloneSourceInfo(info), payload: body, document: placed,
destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: processing,
}) })
s.active[identity] = len(s.stack) s.active[identity] = len(s.stack)
s.visited[identity] = route s.visited[identity] = route
@@ -391,6 +474,195 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
return diagnostics return diagnostics
} }
func (s *importTraversal) sourceDestination(destination Path) Path {
if !s.itemScope {
return destination.clone()
}
return appendDestination(s.itemPrefix, destination)
}
func (s *importTraversal) visitItemSource(canonical string, declaration importDeclaration, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclaration, processing *sourceVersionProcessingResult) []Diagnostic {
parentScope := s.scope
parentItemScope, parentPrefix := s.itemScope, s.itemPrefix
s.scope = acquisitionScope{}
s.itemScope = true
if parentItemScope {
s.itemPrefix = appendDestination(parentPrefix, destination)
} else {
s.itemPrefix = destination.clone()
}
s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(info), payload: body, document: body,
processing: processing,
})
diagnostics := s.walkDeclarations(s.access.dir(canonical), declarations, depth, Path{})
itemScope := s.scope
s.scope, s.itemScope, s.itemPrefix = parentScope, parentItemScope, parentPrefix
if len(diagnostics) != 0 {
return diagnostics
}
documents := make([]*document.Node, 0, len(itemScope.contributions))
for _, contribution := range itemScope.contributions {
documents = append(documents, contribution.document)
}
completed, diagnostics := composeDocuments(documents...)
if len(diagnostics) != 0 {
prefix := destination.Index(0)
if parentItemScope {
prefix = appendDestination(parentPrefix, prefix)
}
return prependDiagnosticPaths(diagnostics, prefix)
}
itemScope.destination = Path{}
completed.acquisition = []acquisitionScope{cloneAcquisitionScope(itemScope)}
projectNestedEvidence(completed, itemScope)
pl := destination
_, placed, err := placeDocumentWithMode(completed.root, pl, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &declaration.location)}
}
placementNodes := destinationWrapperNodeCountForMode(pl, placementModeSequenceItem)
if s.accounting.nodes+placementNodes > s.limits.nodes {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &declaration.location)}
}
s.accounting.nodes += placementNodes
protectionRule := s.rule
if s.versionState != nil && s.versionState.effectiveRule != nil {
protectionRule = s.versionState.effectiveRule
}
if versionNode := fragmentVersionClaim(placed, protectionRule); versionNode != nil {
versionLocation := publicLocation(versionNode.Location)
return []Diagnostic{{
Code: CodeImportVersion, Severity: SeverityError, Path: protectionRule.Path.clone(),
Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation,
}}
}
s.scope.contributions = append(s.scope.contributions, acquiredContribution{
source: cloneSourceInfo(info), payload: completed.root, document: placed,
destination: pl.clone(), relativeDestination: relativeDestination.clone(), processing: processing,
nested: completed,
})
return nil
}
func prependDiagnosticPaths(diagnostics []Diagnostic, prefix Path) []Diagnostic {
cloned := append([]Diagnostic(nil), diagnostics...)
for index := range cloned {
cloned[index].Path = appendDestination(prefix, cloned[index].Path)
cloned[index].Primary = cloneLocation(cloned[index].Primary)
cloned[index].Related = cloneLocations(cloned[index].Related)
}
return cloned
}
func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) {
if composed == nil {
return
}
for sourceIndex, contribution := range scope.contributions {
if contribution.nested == nil {
continue
}
start := 0
for _, assigned := range composed.sequenceRanges {
if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() {
start = assigned.start
break
}
}
prefix := contribution.destination.Index(start)
if sourceIndex < len(composed.applications) {
outer := composed.applications[sourceIndex]
structural := document.Clone(outer.value)
if node, found := documentAtPath(structural, contribution.destination); found && node.Kind == document.Sequence && len(node.Sequence) > start {
node.Sequence[start] = &document.Node{Kind: document.Mapping, Location: node.Sequence[start].Location}
}
outer.value = structural
composed.applications[sourceIndex] = outer
for _, nested := range contribution.nested.applications {
projected := nested
projected.path = appendDestination(prefix, nested.path)
projected.sequences = cloneSequenceHistoryContributions(nested.sequences)
for index := range projected.sequences {
projected.sequences[index].path = appendDestination(prefix, projected.sequences[index].path)
}
composed.applications = append(composed.applications, projected)
}
}
projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix)
}
}
func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) {
var visit func(*document.Node, Path)
visit = func(node *document.Node, path Path) {
if node == nil {
return
}
if origin, ok := origins[path.String()]; ok {
target[appendDestination(prefix, path).String()] = cloneOrigin(origin)
}
for _, entry := range node.Mapping {
visit(entry.Value, path.Key(entry.Key))
}
for index, child := range node.Sequence {
visit(child, path.Index(index))
}
}
visit(root, Path{})
}
func projectAcquiredDestinations(composed *composedDocument, scope acquisitionScope, sources []SourceInfo) {
byID := make(map[string]int, len(sources))
for index := range sources {
byID[sources[index].ID] = index
}
var projectScope func(*composedDocument, acquisitionScope, Path, bool, string)
projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) {
for sourceIndex, contribution := range current.contributions {
if contribution.source.ID == parentSourceID {
continue
}
contributionPrefix := prefix
contributionInsideItem := insideItem
start := 0
for _, assigned := range currentComposition.sequenceRanges {
if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() {
start = assigned.start
break
}
}
if contribution.source.placementMode == placementModeSequenceItem {
container := appendDestination(contributionPrefix, contribution.destination)
contribution.source.Destination = container.clone()
contribution.source.destinationUnresolved = false
current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source)
if position, ok := byID[contribution.source.ID]; ok {
sources[position].Destination = container
sources[position].destinationUnresolved = false
}
contributionPrefix = appendDestination(container, Path{}.Index(start))
contributionInsideItem = true
} else if contributionInsideItem {
exactDestination := appendDestination(contributionPrefix, contribution.destination)
contribution.source.Destination = exactDestination.clone()
contribution.source.destinationUnresolved = false
current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source)
if position, ok := byID[contribution.source.ID]; ok {
sources[position].Destination = exactDestination
sources[position].destinationUnresolved = false
}
}
if contribution.nested != nil && contribution.nested.acquisition != nil {
for _, nestedScope := range contribution.nested.acquisition {
projectScope(contribution.nested, nestedScope, contributionPrefix, contributionInsideItem, contribution.source.ID)
}
}
}
}
projectScope(composed, scope, Path{}, false, "")
}
func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]byte, bool, *Diagnostic) { func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]byte, bool, *Diagnostic) {
remaining := s.limits.bytes - s.accounting.bytes remaining := s.limits.bytes - s.accounting.bytes
limit := min(int64(maxSourceBytes), remaining) limit := min(int64(maxSourceBytes), remaining)

View File

@@ -9,6 +9,8 @@ import (
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
) )
type importTestConfig struct { type importTestConfig struct {
@@ -155,6 +157,8 @@ func TestLoadImportDirectiveValidationAndTemporaryGlobGate(t *testing.T) {
{name: "two targets", body: "configkit: {imports: [{path: a.yml, glob: '*.yml'}]}\n", code: CodeImportInvalid}, {name: "two targets", body: "configkit: {imports: [{path: a.yml, glob: '*.yml'}]}\n", code: CodeImportInvalid},
{name: "unknown declaration field", body: "configkit: {imports: [{path: a.yml, extra: true}]}\n", code: CodeImportInvalid}, {name: "unknown declaration field", body: "configkit: {imports: [{path: a.yml, extra: true}]}\n", code: CodeImportInvalid},
{name: "optional kind", body: "configkit: {imports: [{path: a.yml, optional: 'yes'}]}\n", code: CodeImportInvalid}, {name: "optional kind", body: "configkit: {imports: [{path: a.yml, optional: 'yes'}]}\n", code: CodeImportInvalid},
{name: "placement kind", body: "configkit: {imports: [{path: a.yml, placement: true}]}\n", code: CodeImportInvalid},
{name: "placement value", body: "configkit: {imports: [{path: a.yml, placement: item}]}\n", code: CodeImportInvalid},
{name: "nested directive", body: "nested: {configkit: {}}\n", code: CodeImportInvalid}, {name: "nested directive", body: "nested: {configkit: {}}\n", code: CodeImportInvalid},
} }
for _, test := range tests { for _, test := range tests {
@@ -166,6 +170,322 @@ func TestLoadImportDirectiveValidationAndTemporaryGlobGate(t *testing.T) {
} }
} }
func TestLoadItemImportComposesHelpersAndNestedItemsInDeclaringScope(t *testing.T) {
t.Parallel()
type nested struct {
Value string `yaml:"value"`
}
type record struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
Label string `yaml:"label"`
Metadata map[string]string `yaml:"metadata"`
Children []nested `yaml:"children"`
}
type config struct {
Items []record `yaml:"items"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: records/item.yml
destination: [items]
placement: sequence-item
`)
itemPath := writeImportFile(t, directory, "records/item.yml", `configkit:
imports:
- path: nested/helper.yml
- path: child.yml
destination: [children]
placement: sequence-item
name: authored
`)
helperPath := writeImportFile(t, directory, "records/nested/helper.yml", `configkit:
imports:
- path: more.yml
destination: [metadata]
id: supplied
label: helper
`)
morePath := writeImportFile(t, directory, "records/nested/more.yml", "flavor: transitive\n")
childPath := writeImportFile(t, directory, "records/child.yml", "value: nested\n")
result, err := testLoad[config](File(mainPath), Options[config]{Imports: true})
if err != nil {
t.Fatalf("load item import: %v", err)
}
if len(result.Config.Items) != 1 {
t.Fatalf("items = %#v", result.Config.Items)
}
item := result.Config.Items[0]
if item.ID != "supplied" || item.Name != "authored" || item.Label != "helper" || item.Metadata["flavor"] != "transitive" || len(item.Children) != 1 || item.Children[0].Value != "nested" {
t.Fatalf("composed item = %#v", item)
}
if got := result.Sources(); len(got) != 5 || got[1].Path != canonicalTestPath(t, itemPath) || got[2].Path != canonicalTestPath(t, helperPath) || got[3].Path != canonicalTestPath(t, morePath) || got[4].Path != canonicalTestPath(t, childPath) {
t.Fatalf("source traversal = %#v", got)
}
for _, test := range []struct {
index int
want string
}{
{index: 1, want: "/items"},
{index: 2, want: "/items/0"},
{index: 3, want: "/items/0/metadata"},
{index: 4, want: "/items/0/children"},
} {
if got := result.Sources()[test.index].Destination.String(); got != test.want {
t.Errorf("source %d destination = %q, want %q", test.index+1, got, test.want)
}
}
for _, test := range []struct {
path Path
sourceID string
}{
{path: Keys("items").Index(0).Key("id"), sourceID: "source-3"},
{path: Keys("items").Index(0).Key("metadata").Key("flavor"), sourceID: "source-4"},
} {
info, ok := result.Metadata().Lookup(test.path)
if !ok || !info.Explicit || len(info.Origin.Locations) != 1 || info.Origin.Locations[0].SourceID != test.sourceID {
t.Errorf("origin at %s = %#v, %v, want source %s", test.path.String(), info, ok, test.sourceID)
}
foundInHistory := false
for _, change := range result.Metadata().History(test.path) {
for _, location := range change.Origin.Locations {
foundInHistory = foundInHistory || location.SourceID == test.sourceID
}
}
if !foundInHistory {
t.Errorf("history at %s does not retain source %s", test.path.String(), test.sourceID)
}
}
t.Run("duplicate helper identity remains a mapping conflict", func(t *testing.T) {
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [items]
placement: sequence-item
`)
writeImportFile(t, directory, "item.yml", `configkit:
imports:
- path: helper.yml
id: authored
`)
writeImportFile(t, directory, "helper.yml", "id: supplied\n")
_, err := testLoad[config](File(mainPath), Options[config]{Imports: true})
var loadError *LoadError
if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 {
t.Fatalf("duplicate identity error = %v", err)
}
diagnostic := loadError.Diagnostics()[0]
if diagnostic.Code != CodeComposeConflict || diagnostic.Path.String() != "/items/0/id" || diagnostic.Primary == nil || len(diagnostic.Related) != 1 {
t.Fatalf("duplicate identity diagnostic = %#v", diagnostic)
}
})
}
func TestLayeredUnderlayItemImportRetainsNestedEvidenceAndDestinations(t *testing.T) {
t.Parallel()
type nested struct {
Value string `yaml:"value"`
}
type record struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
Label string `yaml:"label"`
Metadata map[string]string `yaml:"metadata"`
Children []nested `yaml:"children"`
}
type config struct {
Catalog struct {
Entries []record `yaml:"entries"`
} `yaml:"catalog"`
}
directory := t.TempDir()
itemPath := writeImportFile(t, directory, "records/item.yml", `configkit:
imports:
- path: nested/helper.yml
- path: children/child.yml
destination: [children]
placement: sequence-item
name: authored
`)
helperPath := writeImportFile(t, directory, "records/nested/helper.yml", `configkit:
imports:
- path: more.yml
destination: [metadata]
id: supplied
label: helper
`)
morePath := writeImportFile(t, directory, "records/nested/more.yml", "flavor: transitive\n")
childPath := writeImportFile(t, directory, "records/children/child.yml", "value: nested\n")
loaded, err := LoadLayered(
Input{Source: NoSource()}, testSecretBinding(),
LayeredOptions[config]{Underlays: []Underlay{{Inputs: []Input{{
Source: File(itemPath), Imports: true, Destination: Keys("catalog", "entries"), placementMode: placementModeSequenceItem,
}}}}},
)
if err != nil {
if loadError, ok := err.(*LoadError); ok {
t.Fatalf("LoadLayered: %v, diagnostics %#v", err, loadError.Diagnostics())
}
t.Fatalf("LoadLayered: %v", err)
}
result := testLoaded(loaded)
if len(result.Config.Catalog.Entries) != 1 || result.Config.Catalog.Entries[0].ID != "supplied" || result.Config.Catalog.Entries[0].Name != "authored" ||
result.Config.Catalog.Entries[0].Label != "helper" || result.Config.Catalog.Entries[0].Metadata["flavor"] != "transitive" ||
len(result.Config.Catalog.Entries[0].Children) != 1 || result.Config.Catalog.Entries[0].Children[0].Value != "nested" {
t.Fatalf("underlay item = %#v", result.Config.Catalog.Entries)
}
sources := result.Sources()
wantSources := []struct {
path string
destination string
}{
{path: itemPath, destination: "/catalog/entries"},
{path: helperPath, destination: "/catalog/entries/0"},
{path: morePath, destination: "/catalog/entries/0/metadata"},
{path: childPath, destination: "/catalog/entries/0/children"},
}
if len(sources) != len(wantSources) {
t.Fatalf("sources = %#v", sources)
}
for index, want := range wantSources {
if sources[index].Path != canonicalTestPath(t, want.path) || sources[index].Destination.String() != want.destination {
t.Errorf("source %d = %#v, want path %q and destination %q", index+1, sources[index], want.path, want.destination)
}
}
for _, test := range []struct {
path Path
sourceID string
}{
{path: Keys("catalog", "entries").Index(0).Key("id"), sourceID: "source-2"},
{path: Keys("catalog", "entries").Index(0).Key("metadata").Key("flavor"), sourceID: "source-3"},
{path: Keys("catalog", "entries").Index(0).Key("children").Index(0).Key("value"), sourceID: "source-4"},
} {
info, ok := result.Metadata().Lookup(test.path)
if !ok || !info.Explicit || len(info.Origin.Locations) != 1 || info.Origin.Locations[0].SourceID != test.sourceID || info.Origin.Layer != LayerUnderlay {
t.Errorf("origin at %s = %#v, %v, want %s in underlay", test.path.String(), info, ok, test.sourceID)
}
foundInHistory := false
for _, change := range result.Metadata().History(test.path) {
for _, location := range change.Origin.Locations {
foundInHistory = foundInHistory || location.SourceID == test.sourceID
}
}
if !foundInHistory {
t.Errorf("history at %s does not retain source %s", test.path.String(), test.sourceID)
}
}
}
func TestLoadItemGlobKeepsDefaultModeIndependentAndOptionalMissesAbsent(t *testing.T) {
t.Parallel()
type item struct {
Value string `yaml:"value"`
}
type config struct {
Items []item `yaml:"items"`
Other map[string]string `yaml:"other"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- glob: records/*.yml
destination: [items]
placement: sequence-item
- path: absent/*.yml
destination: [items]
optional: true
placement: sequence-item
- path: other.yml
destination: [other]
`)
emptyPath := writeImportFile(t, directory, "records/empty.yml", "{}\n")
writeImportFile(t, directory, "other.yml", "value: mapping-default\n")
result, err := testLoad[config](File(mainPath), Options[config]{Imports: true})
if err != nil {
t.Fatalf("load item glob: %v", err)
}
if len(result.Config.Items) != 1 || result.Config.Items[0].Value != "" || result.Config.Other["value"] != "mapping-default" {
t.Fatalf("config = %#v", result.Config)
}
sources := result.Sources()
if len(sources) != 3 || sources[1].Path != canonicalTestPath(t, emptyPath) || sources[1].Destination.String() != "/items" || sources[1].placementMode != placementModeSequenceItem || sources[2].placementMode != placementModeMapping || sources[2].Destination.String() != "/other" {
t.Fatalf("source placement evidence = %#v", sources)
}
}
func TestLoadMainItemPlacementKeepsItsHelpersInTheItem(t *testing.T) {
t.Parallel()
type item struct {
ID string `yaml:"id"`
Value string `yaml:"value"`
}
type config struct {
Items []item `yaml:"items"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - path: helper.yml\nid: main\n")
helperPath := writeImportFile(t, directory, "helper.yml", "value: helper\n")
result, err := testLoad[config](File(mainPath), Options[config]{
Imports: true, destination: Keys("items"), placementMode: placementModeSequenceItem,
})
if err != nil {
t.Fatalf("load main item: %v", err)
}
if len(result.Config.Items) != 1 || result.Config.Items[0].ID != "main" || result.Config.Items[0].Value != "helper" {
t.Fatalf("main item = %#v", result.Config.Items)
}
sources := result.Sources()
if len(sources) != 2 || sources[0].Destination.String() != "/items" || sources[0].placementMode != placementModeSequenceItem || sources[1].Destination.String() != "/items/0" || sources[1].placementMode != placementModeMapping {
t.Fatalf("main item sources = %#v", sources)
}
if sources[1].Path != canonicalTestPath(t, helperPath) {
t.Fatalf("helper path = %q", sources[1].Path)
}
}
func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T) {
t.Parallel()
type config struct {
Items []map[string]string `yaml:"items"`
}
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `configkit:
imports:
- path: item.yml
destination: [items]
placement: sequence-item
`)
writeImportFile(t, directory, "item.yml", "configkit: {imports: [{path: helper.yml}]}\nname: imported\n")
writeImportFile(t, directory, "helper.yml", "id: helper-id\n")
input, diagnostics := assembleBaseInput(File(mainPath), Options[config]{Imports: true})
if len(diagnostics) != 0 {
t.Fatalf("assemble item import: %#v", diagnostics)
}
authored := parseDocumentForDecode(t, "items: [{id: authored-id, name: authored}]\n")
composed, diagnostics := composeDocumentsWithPolicy(sequencePolicy{identityFields: map[string]string{"/items": "id"}}, authored, input.document)
if len(diagnostics) != 0 {
t.Fatalf("compose authored and imported lists: %#v", diagnostics)
}
items, ok := documentAtPath(composed.root, Keys("items"))
if !ok || items.Kind != document.Sequence || len(items.Sequence) != 2 {
t.Fatalf("composed items = %#v", items)
}
if id, ok := documentAtPath(composed.root, Keys("items").Index(1).Key("id")); !ok || id.Text != "helper-id" {
t.Fatalf("imported helper ID = %#v, %v", id, ok)
}
if origin := composed.origins[Keys("items").Index(1).Key("id").String()]; len(origin.Locations) != 1 || origin.Locations[0].SourceID != "source-3" {
t.Fatalf("registered composition origin = %#v", origin)
}
}
func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) { func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) {
t.Parallel() t.Parallel()
@@ -201,6 +521,19 @@ func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) {
} }
} }
func TestParseImportPlacementDefaultsPerDeclaration(t *testing.T) {
t.Parallel()
declarations := parseImportDeclarationsYAML(t, `configkit:
imports:
- path: mapping.yml
- glob: records/*.yml
placement: sequence-item
`)
if len(declarations) != 2 || declarations[0].placement != placementModeMapping || declarations[1].placement != placementModeSequenceItem {
t.Fatalf("declaration placements = %#v", declarations)
}
}
func TestParseImportDestinationRejectsInvalidShapesAndMembers(t *testing.T) { func TestParseImportDestinationRejectsInvalidShapesAndMembers(t *testing.T) {
t.Parallel() t.Parallel()

View File

@@ -28,6 +28,7 @@ type acquiredContribution struct {
destination Path destination Path
relativeDestination Path relativeDestination Path
processing *sourceVersionProcessingResult processing *sourceVersionProcessingResult
nested *composedDocument
} }
// acquisitionScope is the ordered strict mapping scope produced by one input // acquisitionScope is the ordered strict mapping scope produced by one input
@@ -46,6 +47,7 @@ func cloneAcquisitionScope(scope acquisitionScope) acquisitionScope {
source: cloneSourceInfo(contribution.source), payload: contribution.payload, source: cloneSourceInfo(contribution.source), payload: contribution.payload,
document: contribution.document, destination: contribution.destination.clone(), document: contribution.document, destination: contribution.destination.clone(),
relativeDestination: contribution.relativeDestination.clone(), processing: contribution.processing, relativeDestination: contribution.relativeDestination.clone(), processing: contribution.processing,
nested: cloneComposedDocument(contribution.nested),
} }
} }
} }

View File

@@ -201,6 +201,12 @@ func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRu
composition.inputBytes = accounting.bytes - initialBytes composition.inputBytes = accounting.bytes - initialBytes
composition.inputNodes = accounting.nodes - initialNodes composition.inputNodes = accounting.nodes - initialNodes
composition.acquisition = cloneAcquisitionScopes(scopes) composition.acquisition = cloneAcquisitionScopes(scopes)
var projectionScope acquisitionScope
for _, scope := range scopes {
projectionScope.contributions = append(projectionScope.contributions, scope.contributions...)
}
projectNestedEvidence(composition, projectionScope)
projectAcquiredDestinations(composition, projectionScope, sources)
for pointer, origin := range composition.origins { for pointer, origin := range composition.origins {
origin.Layer = LayerUnderlay origin.Layer = LayerUnderlay
composition.origins[pointer] = origin composition.origins[pointer] = origin

View File

@@ -42,6 +42,34 @@ items: {main: main}
} }
} }
func TestNativeFilesystemItemImportUsesSharedScopedAssembly(t *testing.T) {
t.Parallel()
type item struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
}
type config struct {
Items []item `yaml:"items"`
}
filesystem := fstest.MapFS{
"main.yml": &fstest.MapFile{Data: []byte("configkit:\n imports:\n - path: records/item.yml\n destination: [items]\n placement: sequence-item\n")},
"records/item.yml": &fstest.MapFile{Data: []byte("configkit:\n imports:\n - path: helper.yml\nid: native\n")},
"records/helper.yml": &fstest.MapFile{Data: []byte("name: helper\n")},
}
namespace := newNativeNamespace("bundle", filesystem)
result, err := testLoad[config](filesystemFile(namespace, "main.yml"), Options[config]{Imports: true})
if err != nil {
t.Fatalf("native item import: %v", err)
}
if len(result.Config.Items) != 1 || result.Config.Items[0].ID != "native" || result.Config.Items[0].Name != "helper" {
t.Fatalf("native item = %#v", result.Config.Items)
}
sources := result.Sources()
if len(sources) != 3 || sources[1].Destination.String() != "/items" || sources[2].Destination.String() != "/items/0" || sources[1].Path != "records/item.yml" || sources[2].Path != "records/helper.yml" {
t.Fatalf("native item sources = %#v", sources)
}
}
func TestNativeFilesystemNamespaceAndPathFailures(t *testing.T) { func TestNativeFilesystemNamespaceAndPathFailures(t *testing.T) {
first := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte("items: {name: first}\n")}} first := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte("items: {name: first}\n")}}
second := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte("items: {name: second}\n")}} second := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte("items: {name: second}\n")}}