From 25cf6d32b60eca93e010d8f6690e0ddd5525d878 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 16 Oct 2026 16:20:14 +0000 Subject: [PATCH] Assemble imported sequence-item scopes --- docs/config.md | 19 ++ docs/roadmap/implementation.md | 16 +- internal/engine/compose.go | 3 + internal/engine/directives.go | 30 +- internal/engine/imports.go | 312 ++++++++++++++++++-- internal/engine/imports_test.go | 333 ++++++++++++++++++++++ internal/engine/load_input.go | 2 + internal/engine/load_request.go | 6 + internal/engine/native_filesystem_test.go | 28 ++ 9 files changed, 722 insertions(+), 27 deletions(-) diff --git a/docs/config.md b/docs/config.md index 87dcc8d..47eb2dd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -884,6 +884,25 @@ walk failures, matched directories, invalid documents, escapes, and repeated tar remain errors. Final candidates are bounded by the remaining source limit before their contents are accumulated. +Each path or glob declaration may include `placement: mapping` or +`placement: sequence-item`. Omission means `mapping` for that declaration, +including every match of a glob; placement is not inherited. Sequence-item +placement requires a nonempty effective destination and wraps the imported +mapping as one element of a sequence there. It does not split source sequences +or enable sequence accumulation: overlapping sequences still follow the strict +composition rules above. + +An item import first opens a local mapping scope. Its body and ordinary helper +imports compose within that record. An omitted helper destination or +`destination: []` places the helper at the record root; other destinations are +relative to the immediate importing source. A helper may supply fields such as +the record's `id`. An explicit nested `sequence-item` import creates a sequence +inside the record, and its placement is independent of the enclosing item. +At the record root, an item import must name a nonempty nested destination. +Physical paths continue to resolve from the file that declares each import. +An actual empty mapping contributes an empty item, while an optional unmatched +path or glob contributes nothing. + For host files, the configuration root defaults to the selected file's canonical directory. A host file may supply `FilesystemContext.Root` to widen that root but may not supply `BaseDir` or a native `FS`. Byte, reader, and mapping inputs with host context diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index bc62ecd..40ca605 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Implementation Plan: Sequence Placement and Composition -Status: implementation in progress; stages 1–7 completed. +Status: implementation in progress; stages 1–8 completed. ## Purpose and authority @@ -505,6 +505,20 @@ Validation: import/directive/glob/destination/composition tests and `./scripts/check.sh`. Completion: private item imports construct the agreed records and preserve physical acquisition evidence. +Completion evidence: import declarations now parse per-declaration placement, +and item imports compose their body and ordinary helpers in a local mapping +scope before wrapping the completed record. Nested item scopes, glob matches, +relative and transitive helper destinations, source origins/history, and +resolved underlay destinations are retained. Configuration documentation +describes only the implemented import-placement subset; private registration +controls remain unadvertised. These checks passed with Go `go1.26.9 linux/amd64`: + +- `GOTOOLCHAIN=local GOPROXY=off go test ./internal/engine -run 'Import|Directive|Glob|Destination|Compose|NativeFilesystem' -count=1` +- `GOTOOLCHAIN=local ./scripts/check.sh` +- `git diff --check` + +No environmental limitation affected validation; checks ran on Linux/amd64. + ## Stage 9 — Wire strict policy through the complete loading lifecycle Scope: ordinary/grouped engine loading and every strict-tier entry point. diff --git a/internal/engine/compose.go b/internal/engine/compose.go index 1cf4b4d..29ba829 100644 --- a/internal/engine/compose.go +++ b/internal/engine/compose.go @@ -45,6 +45,9 @@ func newAssemblyAccumulator(owned *composedDocument) *assemblyAccumulator { // cloneComposedDocument detaches mutable composition and provenance state; // history application trees are immutable and retain their existing sharing. func cloneComposedDocument(base *composedDocument) *composedDocument { + if base == nil { + return nil + } return &composedDocument{ root: document.Clone(base.root), claims: cloneCompositionClaims(base.claims), acquisition: cloneAcquisitionScopes(base.acquisition), diff --git a/internal/engine/directives.go b/internal/engine/directives.go index 61f6079..f4a6724 100644 --- a/internal/engine/directives.go +++ b/internal/engine/directives.go @@ -10,11 +10,14 @@ import ( const directiveKey = "configkit" type importDeclaration struct { - path string - glob string - destination Path - optional bool - location Location + path string + glob string + destination Path + placement placementMode + placementPath Path + placementLocation *Location + optional bool + location Location } func splitImportDirectives(root *document.Node) (*document.Node, []importDeclaration, []Diagnostic) { @@ -70,7 +73,7 @@ func parseImportDeclaration(node *document.Node, declarationPath Path) (importDe if node.Kind != document.Mapping { return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, declarationPath, "import declaration must be a mapping", node)} } - declaration := importDeclaration{location: publicLocation(node.Location)} + declaration := importDeclaration{location: publicLocation(node.Location), placementPath: declarationPath.clone()} seenTarget := false for _, entry := range node.Mapping { fieldPath := declarationPath.Key(entry.Key) @@ -112,6 +115,21 @@ func parseImportDeclaration(node *document.Node, declarationPath Path) (importDe return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "destination must contain only non-reserved mapping keys", entry.Value)} } declaration.destination = normalized + case "placement": + declaration.placementPath = fieldPath.clone() + placementLocation := publicLocation(entry.Value.Location) + declaration.placementLocation = &placementLocation + if entry.Value.Kind != document.String { + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)} + } + switch entry.Value.Text { + case "mapping": + declaration.placement = placementModeMapping + case "sequence-item": + declaration.placement = placementModeSequenceItem + default: + return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "placement must be mapping or sequence-item", entry.Value)} + } default: return importDeclaration{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, fieldPath, "import declaration field is not recognized", entry.Value)} } diff --git a/internal/engine/imports.go b/internal/engine/imports.go index 06af12e..395831a 100644 --- a/internal/engine/imports.go +++ b/internal/engine/imports.go @@ -51,11 +51,13 @@ type importTraversal struct { rule *VersionRule versionState *sourceVersionState - scope acquisitionScope - sources []SourceInfo - active map[resourceIdentity]int - visited map[resourceIdentity]importRoute - stack []importRoute + scope acquisitionScope + itemScope bool + itemPrefix Path + sources []SourceInfo + active map[resourceIdentity]int + visited map[resourceIdentity]importRoute + stack []importRoute } func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { @@ -128,6 +130,10 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa } mainInfo := cloneSourceInfo(*main.source) mainInfo.Destination = destination.clone() + mainIsItem := main.source.placementMode == placementModeSequenceItem + if mainIsItem { + mainInfo.destinationUnresolved = true + } if mainPath != "" { mainInfo.Path = mainPath } @@ -148,6 +154,12 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa sources: []SourceInfo{mainInfo}, active: make(map[resourceIdentity]int), visited: visited, } + if mainIsItem { + state.itemScope = true + state.itemPrefix = destination.clone() + state.scope.destination = Path{} + state.sources[0] = cloneSourceInfo(mainInfo) + } if state.visited == nil { state.visited = make(map[resourceIdentity]importRoute) } @@ -174,19 +186,26 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa body, declarations, diagnostics = splitImportDirectives(main.document) } if len(diagnostics) == 0 { - _, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) - placementNodes := destinationWrapperNodeCount(destination) - if err != nil { - diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} - } else if accounting.nodes+placementNodes > limits.nodes { - diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} - } else { - accounting.nodes += placementNodes + if mainIsItem { state.scope.contributions = append(state.scope.contributions, acquiredContribution{ - source: cloneSourceInfo(mainInfo), payload: body, document: placed, - destination: destination.clone(), processing: processing, + source: cloneSourceInfo(mainInfo), payload: body, document: body, processing: processing, }) - diagnostics = state.walkDeclarations(basePath, declarations, 0, destination) + diagnostics = state.walkDeclarations(basePath, declarations, 0, Path{}) + } else { + _, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) + placementNodes := destinationWrapperNodeCount(destination) + if err != nil { + diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} + } else if accounting.nodes+placementNodes > limits.nodes { + diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} + } else { + accounting.nodes += placementNodes + state.scope.contributions = append(state.scope.contributions, acquiredContribution{ + source: cloneSourceInfo(mainInfo), payload: body, document: placed, + destination: destination.clone(), processing: processing, + }) + diagnostics = state.walkDeclarations(basePath, declarations, 0, destination) + } } } @@ -197,12 +216,49 @@ func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePa if closeErr != nil { return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)} } + if mainIsItem { + localScope := state.scope + documents := make([]*document.Node, 0, len(localScope.contributions)) + for _, contribution := range localScope.contributions { + documents = append(documents, contribution.document) + } + completed, failures := composeDocuments(documents...) + if len(failures) != 0 { + return nil, cloneSourceInfos(state.sources), prependDiagnosticPaths(failures, destination.Index(0)) + } + completed.acquisition = []acquisitionScope{cloneAcquisitionScope(localScope)} + projectNestedEvidence(completed, localScope) + _, placed, err := placeDocumentWithMode(completed.root, destination, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes) + if err != nil { + return nil, cloneSourceInfos(state.sources), []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")} + } + placementNodes := destinationWrapperNodeCountForMode(destination, placementModeSequenceItem) + if accounting.nodes+placementNodes > limits.nodes { + return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)} + } + accounting.nodes += placementNodes + composed, failures := composeDocuments(placed) + if composed != nil { + synthetic := acquisitionScope{contributions: []acquiredContribution{{ + source: cloneSourceInfo(mainInfo), payload: completed.root, document: placed, + destination: destination.clone(), processing: processing, nested: completed, + }}} + projectNestedEvidence(composed, synthetic) + projectAcquiredDestinations(composed, synthetic, state.sources) + composed.acquisition = []acquisitionScope{cloneAcquisitionScope(synthetic)} + composed.inputBytes = accounting.bytes + composed.inputNodes = accounting.nodes + } + return composed, cloneSourceInfos(state.sources), failures + } documents := make([]*document.Node, 0, len(state.scope.contributions)) for _, contribution := range state.scope.contributions { documents = append(documents, contribution.document) } composed, diagnostics := composeDocuments(documents...) if composed != nil { + projectNestedEvidence(composed, state.scope) + projectAcquiredDestinations(composed, state.scope, state.sources) composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)} composed.inputBytes = accounting.bytes composed.inputNodes = accounting.nodes @@ -260,6 +316,14 @@ func (s *importTraversal) walkDeclarations(basePath string, declarations []impor func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, destination, relativeDestination Path, depth int) []Diagnostic { location := declaration.location + if declaration.placement == placementModeSequenceItem && len(destination.segments) == 0 { + return []Diagnostic{{ + Code: CodeImportInvalid, Severity: SeverityError, + Path: declaration.placementPath.clone(), + Message: "sequence-item placement requires a nonempty destination", + Primary: cloneLocation(declaration.placementLocation), + }} + } if depth > s.limits.depth { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)} } @@ -312,7 +376,11 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati kind = SourceKindFilesystem label = s.access.label + "/" + canonical } - info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: destination.clone(), Layer: LayerBase, Kind: kind, ImportedFrom: &location} + info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: s.sourceDestination(destination), Layer: LayerBase, Kind: kind, ImportedFrom: &location} + if declaration.placement == placementModeSequenceItem { + info.placementMode = placementModeSequenceItem + } + info.destinationUnresolved = s.itemScope || declaration.placement == placementModeSequenceItem if kind == SourceKindFilesystem { info.namespace = s.access.token } @@ -356,11 +424,26 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati return diagnostics } } - _, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes) + placementDestination := destination + mode := placementModeMapping + if declaration.placement == placementModeSequenceItem { + mode = placementModeSequenceItem + } + if mode == placementModeSequenceItem { + route := importRoute{canonical: canonical, locations: currentRoute.locations} + s.active[identity] = len(s.stack) + s.visited[identity] = route + s.stack = append(s.stack, route) + diagnostics := s.visitItemSource(canonical, declaration, destination, relativeDestination, depth, info, body, declarations, processing) + delete(s.active, identity) + s.stack = s.stack[:len(s.stack)-1] + return diagnostics + } + _, placed, err := placeDocument(body, placementDestination, body.Location, maxDocumentDepth, maxDocumentNodes) if err != nil { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)} } - placementNodes := destinationWrapperNodeCount(destination) + placementNodes := destinationWrapperNodeCount(placementDestination) if s.accounting.nodes+placementNodes > s.limits.nodes { return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)} } @@ -380,7 +463,7 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati route := importRoute{canonical: canonical, locations: currentRoute.locations} s.scope.contributions = append(s.scope.contributions, acquiredContribution{ source: cloneSourceInfo(info), payload: body, document: placed, - destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, + destination: placementDestination.clone(), relativeDestination: relativeDestination.clone(), processing: processing, }) s.active[identity] = len(s.stack) s.visited[identity] = route @@ -391,6 +474,195 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati return diagnostics } +func (s *importTraversal) sourceDestination(destination Path) Path { + if !s.itemScope { + return destination.clone() + } + return appendDestination(s.itemPrefix, destination) +} + +func (s *importTraversal) visitItemSource(canonical string, declaration importDeclaration, destination, relativeDestination Path, depth int, info SourceInfo, body *document.Node, declarations []importDeclaration, processing *sourceVersionProcessingResult) []Diagnostic { + parentScope := s.scope + parentItemScope, parentPrefix := s.itemScope, s.itemPrefix + s.scope = acquisitionScope{} + s.itemScope = true + if parentItemScope { + s.itemPrefix = appendDestination(parentPrefix, destination) + } else { + s.itemPrefix = destination.clone() + } + s.scope.contributions = append(s.scope.contributions, acquiredContribution{ + source: cloneSourceInfo(info), payload: body, document: body, + processing: processing, + }) + diagnostics := s.walkDeclarations(s.access.dir(canonical), declarations, depth, Path{}) + itemScope := s.scope + s.scope, s.itemScope, s.itemPrefix = parentScope, parentItemScope, parentPrefix + if len(diagnostics) != 0 { + return diagnostics + } + documents := make([]*document.Node, 0, len(itemScope.contributions)) + for _, contribution := range itemScope.contributions { + documents = append(documents, contribution.document) + } + completed, diagnostics := composeDocuments(documents...) + if len(diagnostics) != 0 { + prefix := destination.Index(0) + if parentItemScope { + prefix = appendDestination(parentPrefix, prefix) + } + return prependDiagnosticPaths(diagnostics, prefix) + } + itemScope.destination = Path{} + completed.acquisition = []acquisitionScope{cloneAcquisitionScope(itemScope)} + projectNestedEvidence(completed, itemScope) + pl := destination + _, placed, err := placeDocumentWithMode(completed.root, pl, placementModeSequenceItem, body.Location, maxDocumentDepth, maxDocumentNodes) + if err != nil { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &declaration.location)} + } + placementNodes := destinationWrapperNodeCountForMode(pl, placementModeSequenceItem) + if s.accounting.nodes+placementNodes > s.limits.nodes { + return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &declaration.location)} + } + s.accounting.nodes += placementNodes + protectionRule := s.rule + if s.versionState != nil && s.versionState.effectiveRule != nil { + protectionRule = s.versionState.effectiveRule + } + if versionNode := fragmentVersionClaim(placed, protectionRule); versionNode != nil { + versionLocation := publicLocation(versionNode.Location) + return []Diagnostic{{ + Code: CodeImportVersion, Severity: SeverityError, Path: protectionRule.Path.clone(), + Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation, + }} + } + s.scope.contributions = append(s.scope.contributions, acquiredContribution{ + source: cloneSourceInfo(info), payload: completed.root, document: placed, + destination: pl.clone(), relativeDestination: relativeDestination.clone(), processing: processing, + nested: completed, + }) + return nil +} + +func prependDiagnosticPaths(diagnostics []Diagnostic, prefix Path) []Diagnostic { + cloned := append([]Diagnostic(nil), diagnostics...) + for index := range cloned { + cloned[index].Path = appendDestination(prefix, cloned[index].Path) + cloned[index].Primary = cloneLocation(cloned[index].Primary) + cloned[index].Related = cloneLocations(cloned[index].Related) + } + return cloned +} + +func projectNestedEvidence(composed *composedDocument, scope acquisitionScope) { + if composed == nil { + return + } + for sourceIndex, contribution := range scope.contributions { + if contribution.nested == nil { + continue + } + start := 0 + for _, assigned := range composed.sequenceRanges { + if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() { + start = assigned.start + break + } + } + prefix := contribution.destination.Index(start) + if sourceIndex < len(composed.applications) { + outer := composed.applications[sourceIndex] + structural := document.Clone(outer.value) + if node, found := documentAtPath(structural, contribution.destination); found && node.Kind == document.Sequence && len(node.Sequence) > start { + node.Sequence[start] = &document.Node{Kind: document.Mapping, Location: node.Sequence[start].Location} + } + outer.value = structural + composed.applications[sourceIndex] = outer + for _, nested := range contribution.nested.applications { + projected := nested + projected.path = appendDestination(prefix, nested.path) + projected.sequences = cloneSequenceHistoryContributions(nested.sequences) + for index := range projected.sequences { + projected.sequences[index].path = appendDestination(prefix, projected.sequences[index].path) + } + composed.applications = append(composed.applications, projected) + } + } + projectNestedOrigins(composed.origins, contribution.nested.root, contribution.nested.origins, prefix) + } +} + +func projectNestedOrigins(target map[string]Origin, root *document.Node, origins map[string]Origin, prefix Path) { + var visit func(*document.Node, Path) + visit = func(node *document.Node, path Path) { + if node == nil { + return + } + if origin, ok := origins[path.String()]; ok { + target[appendDestination(prefix, path).String()] = cloneOrigin(origin) + } + for _, entry := range node.Mapping { + visit(entry.Value, path.Key(entry.Key)) + } + for index, child := range node.Sequence { + visit(child, path.Index(index)) + } + } + visit(root, Path{}) +} + +func projectAcquiredDestinations(composed *composedDocument, scope acquisitionScope, sources []SourceInfo) { + byID := make(map[string]int, len(sources)) + for index := range sources { + byID[sources[index].ID] = index + } + var projectScope func(*composedDocument, acquisitionScope, Path, bool, string) + projectScope = func(currentComposition *composedDocument, current acquisitionScope, prefix Path, insideItem bool, parentSourceID string) { + for sourceIndex, contribution := range current.contributions { + if contribution.source.ID == parentSourceID { + continue + } + contributionPrefix := prefix + contributionInsideItem := insideItem + start := 0 + for _, assigned := range currentComposition.sequenceRanges { + if assigned.sourceIndex == sourceIndex && assigned.path.String() == contribution.destination.String() { + start = assigned.start + break + } + } + if contribution.source.placementMode == placementModeSequenceItem { + container := appendDestination(contributionPrefix, contribution.destination) + contribution.source.Destination = container.clone() + contribution.source.destinationUnresolved = false + current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source) + if position, ok := byID[contribution.source.ID]; ok { + sources[position].Destination = container + sources[position].destinationUnresolved = false + } + contributionPrefix = appendDestination(container, Path{}.Index(start)) + contributionInsideItem = true + } else if contributionInsideItem { + exactDestination := appendDestination(contributionPrefix, contribution.destination) + contribution.source.Destination = exactDestination.clone() + contribution.source.destinationUnresolved = false + current.contributions[sourceIndex].source = cloneSourceInfo(contribution.source) + if position, ok := byID[contribution.source.ID]; ok { + sources[position].Destination = exactDestination + sources[position].destinationUnresolved = false + } + } + if contribution.nested != nil && contribution.nested.acquisition != nil { + for _, nestedScope := range contribution.nested.acquisition { + projectScope(contribution.nested, nestedScope, contributionPrefix, contributionInsideItem, contribution.source.ID) + } + } + } + } + projectScope(composed, scope, Path{}, false, "") +} + func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]byte, bool, *Diagnostic) { remaining := s.limits.bytes - s.accounting.bytes limit := min(int64(maxSourceBytes), remaining) diff --git a/internal/engine/imports_test.go b/internal/engine/imports_test.go index 821d37c..953ff97 100644 --- a/internal/engine/imports_test.go +++ b/internal/engine/imports_test.go @@ -9,6 +9,8 @@ import ( "strconv" "strings" "testing" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" ) type importTestConfig struct { @@ -155,6 +157,8 @@ func TestLoadImportDirectiveValidationAndTemporaryGlobGate(t *testing.T) { {name: "two targets", body: "configkit: {imports: [{path: a.yml, glob: '*.yml'}]}\n", code: CodeImportInvalid}, {name: "unknown declaration field", body: "configkit: {imports: [{path: a.yml, extra: true}]}\n", code: CodeImportInvalid}, {name: "optional kind", body: "configkit: {imports: [{path: a.yml, optional: 'yes'}]}\n", code: CodeImportInvalid}, + {name: "placement kind", body: "configkit: {imports: [{path: a.yml, placement: true}]}\n", code: CodeImportInvalid}, + {name: "placement value", body: "configkit: {imports: [{path: a.yml, placement: item}]}\n", code: CodeImportInvalid}, {name: "nested directive", body: "nested: {configkit: {}}\n", code: CodeImportInvalid}, } for _, test := range tests { @@ -166,6 +170,322 @@ func TestLoadImportDirectiveValidationAndTemporaryGlobGate(t *testing.T) { } } +func TestLoadItemImportComposesHelpersAndNestedItemsInDeclaringScope(t *testing.T) { + t.Parallel() + type nested struct { + Value string `yaml:"value"` + } + type record struct { + ID string `yaml:"id"` + Name string `yaml:"name"` + Label string `yaml:"label"` + Metadata map[string]string `yaml:"metadata"` + Children []nested `yaml:"children"` + } + type config struct { + Items []record `yaml:"items"` + } + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - path: records/item.yml + destination: [items] + placement: sequence-item +`) + itemPath := writeImportFile(t, directory, "records/item.yml", `configkit: + imports: + - path: nested/helper.yml + - path: child.yml + destination: [children] + placement: sequence-item +name: authored +`) + helperPath := writeImportFile(t, directory, "records/nested/helper.yml", `configkit: + imports: + - path: more.yml + destination: [metadata] +id: supplied +label: helper +`) + morePath := writeImportFile(t, directory, "records/nested/more.yml", "flavor: transitive\n") + childPath := writeImportFile(t, directory, "records/child.yml", "value: nested\n") + + result, err := testLoad[config](File(mainPath), Options[config]{Imports: true}) + if err != nil { + t.Fatalf("load item import: %v", err) + } + if len(result.Config.Items) != 1 { + t.Fatalf("items = %#v", result.Config.Items) + } + item := result.Config.Items[0] + if item.ID != "supplied" || item.Name != "authored" || item.Label != "helper" || item.Metadata["flavor"] != "transitive" || len(item.Children) != 1 || item.Children[0].Value != "nested" { + t.Fatalf("composed item = %#v", item) + } + if got := result.Sources(); len(got) != 5 || got[1].Path != canonicalTestPath(t, itemPath) || got[2].Path != canonicalTestPath(t, helperPath) || got[3].Path != canonicalTestPath(t, morePath) || got[4].Path != canonicalTestPath(t, childPath) { + t.Fatalf("source traversal = %#v", got) + } + for _, test := range []struct { + index int + want string + }{ + {index: 1, want: "/items"}, + {index: 2, want: "/items/0"}, + {index: 3, want: "/items/0/metadata"}, + {index: 4, want: "/items/0/children"}, + } { + if got := result.Sources()[test.index].Destination.String(); got != test.want { + t.Errorf("source %d destination = %q, want %q", test.index+1, got, test.want) + } + } + for _, test := range []struct { + path Path + sourceID string + }{ + {path: Keys("items").Index(0).Key("id"), sourceID: "source-3"}, + {path: Keys("items").Index(0).Key("metadata").Key("flavor"), sourceID: "source-4"}, + } { + info, ok := result.Metadata().Lookup(test.path) + if !ok || !info.Explicit || len(info.Origin.Locations) != 1 || info.Origin.Locations[0].SourceID != test.sourceID { + t.Errorf("origin at %s = %#v, %v, want source %s", test.path.String(), info, ok, test.sourceID) + } + foundInHistory := false + for _, change := range result.Metadata().History(test.path) { + for _, location := range change.Origin.Locations { + foundInHistory = foundInHistory || location.SourceID == test.sourceID + } + } + if !foundInHistory { + t.Errorf("history at %s does not retain source %s", test.path.String(), test.sourceID) + } + } + + t.Run("duplicate helper identity remains a mapping conflict", func(t *testing.T) { + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - path: item.yml + destination: [items] + placement: sequence-item +`) + writeImportFile(t, directory, "item.yml", `configkit: + imports: + - path: helper.yml +id: authored +`) + writeImportFile(t, directory, "helper.yml", "id: supplied\n") + _, err := testLoad[config](File(mainPath), Options[config]{Imports: true}) + var loadError *LoadError + if !errors.As(err, &loadError) || len(loadError.Diagnostics()) != 1 { + t.Fatalf("duplicate identity error = %v", err) + } + diagnostic := loadError.Diagnostics()[0] + if diagnostic.Code != CodeComposeConflict || diagnostic.Path.String() != "/items/0/id" || diagnostic.Primary == nil || len(diagnostic.Related) != 1 { + t.Fatalf("duplicate identity diagnostic = %#v", diagnostic) + } + }) +} + +func TestLayeredUnderlayItemImportRetainsNestedEvidenceAndDestinations(t *testing.T) { + t.Parallel() + type nested struct { + Value string `yaml:"value"` + } + type record struct { + ID string `yaml:"id"` + Name string `yaml:"name"` + Label string `yaml:"label"` + Metadata map[string]string `yaml:"metadata"` + Children []nested `yaml:"children"` + } + type config struct { + Catalog struct { + Entries []record `yaml:"entries"` + } `yaml:"catalog"` + } + + directory := t.TempDir() + itemPath := writeImportFile(t, directory, "records/item.yml", `configkit: + imports: + - path: nested/helper.yml + - path: children/child.yml + destination: [children] + placement: sequence-item +name: authored +`) + helperPath := writeImportFile(t, directory, "records/nested/helper.yml", `configkit: + imports: + - path: more.yml + destination: [metadata] +id: supplied +label: helper +`) + morePath := writeImportFile(t, directory, "records/nested/more.yml", "flavor: transitive\n") + childPath := writeImportFile(t, directory, "records/children/child.yml", "value: nested\n") + + loaded, err := LoadLayered( + Input{Source: NoSource()}, testSecretBinding(), + LayeredOptions[config]{Underlays: []Underlay{{Inputs: []Input{{ + Source: File(itemPath), Imports: true, Destination: Keys("catalog", "entries"), placementMode: placementModeSequenceItem, + }}}}}, + ) + if err != nil { + if loadError, ok := err.(*LoadError); ok { + t.Fatalf("LoadLayered: %v, diagnostics %#v", err, loadError.Diagnostics()) + } + t.Fatalf("LoadLayered: %v", err) + } + result := testLoaded(loaded) + if len(result.Config.Catalog.Entries) != 1 || result.Config.Catalog.Entries[0].ID != "supplied" || result.Config.Catalog.Entries[0].Name != "authored" || + result.Config.Catalog.Entries[0].Label != "helper" || result.Config.Catalog.Entries[0].Metadata["flavor"] != "transitive" || + len(result.Config.Catalog.Entries[0].Children) != 1 || result.Config.Catalog.Entries[0].Children[0].Value != "nested" { + t.Fatalf("underlay item = %#v", result.Config.Catalog.Entries) + } + + sources := result.Sources() + wantSources := []struct { + path string + destination string + }{ + {path: itemPath, destination: "/catalog/entries"}, + {path: helperPath, destination: "/catalog/entries/0"}, + {path: morePath, destination: "/catalog/entries/0/metadata"}, + {path: childPath, destination: "/catalog/entries/0/children"}, + } + if len(sources) != len(wantSources) { + t.Fatalf("sources = %#v", sources) + } + for index, want := range wantSources { + if sources[index].Path != canonicalTestPath(t, want.path) || sources[index].Destination.String() != want.destination { + t.Errorf("source %d = %#v, want path %q and destination %q", index+1, sources[index], want.path, want.destination) + } + } + + for _, test := range []struct { + path Path + sourceID string + }{ + {path: Keys("catalog", "entries").Index(0).Key("id"), sourceID: "source-2"}, + {path: Keys("catalog", "entries").Index(0).Key("metadata").Key("flavor"), sourceID: "source-3"}, + {path: Keys("catalog", "entries").Index(0).Key("children").Index(0).Key("value"), sourceID: "source-4"}, + } { + info, ok := result.Metadata().Lookup(test.path) + if !ok || !info.Explicit || len(info.Origin.Locations) != 1 || info.Origin.Locations[0].SourceID != test.sourceID || info.Origin.Layer != LayerUnderlay { + t.Errorf("origin at %s = %#v, %v, want %s in underlay", test.path.String(), info, ok, test.sourceID) + } + foundInHistory := false + for _, change := range result.Metadata().History(test.path) { + for _, location := range change.Origin.Locations { + foundInHistory = foundInHistory || location.SourceID == test.sourceID + } + } + if !foundInHistory { + t.Errorf("history at %s does not retain source %s", test.path.String(), test.sourceID) + } + } +} + +func TestLoadItemGlobKeepsDefaultModeIndependentAndOptionalMissesAbsent(t *testing.T) { + t.Parallel() + type item struct { + Value string `yaml:"value"` + } + type config struct { + Items []item `yaml:"items"` + Other map[string]string `yaml:"other"` + } + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - glob: records/*.yml + destination: [items] + placement: sequence-item + - path: absent/*.yml + destination: [items] + optional: true + placement: sequence-item + - path: other.yml + destination: [other] +`) + emptyPath := writeImportFile(t, directory, "records/empty.yml", "{}\n") + writeImportFile(t, directory, "other.yml", "value: mapping-default\n") + result, err := testLoad[config](File(mainPath), Options[config]{Imports: true}) + if err != nil { + t.Fatalf("load item glob: %v", err) + } + if len(result.Config.Items) != 1 || result.Config.Items[0].Value != "" || result.Config.Other["value"] != "mapping-default" { + t.Fatalf("config = %#v", result.Config) + } + sources := result.Sources() + if len(sources) != 3 || sources[1].Path != canonicalTestPath(t, emptyPath) || sources[1].Destination.String() != "/items" || sources[1].placementMode != placementModeSequenceItem || sources[2].placementMode != placementModeMapping || sources[2].Destination.String() != "/other" { + t.Fatalf("source placement evidence = %#v", sources) + } +} + +func TestLoadMainItemPlacementKeepsItsHelpersInTheItem(t *testing.T) { + t.Parallel() + type item struct { + ID string `yaml:"id"` + Value string `yaml:"value"` + } + type config struct { + Items []item `yaml:"items"` + } + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", "configkit:\n imports:\n - path: helper.yml\nid: main\n") + helperPath := writeImportFile(t, directory, "helper.yml", "value: helper\n") + result, err := testLoad[config](File(mainPath), Options[config]{ + Imports: true, destination: Keys("items"), placementMode: placementModeSequenceItem, + }) + if err != nil { + t.Fatalf("load main item: %v", err) + } + if len(result.Config.Items) != 1 || result.Config.Items[0].ID != "main" || result.Config.Items[0].Value != "helper" { + t.Fatalf("main item = %#v", result.Config.Items) + } + sources := result.Sources() + if len(sources) != 2 || sources[0].Destination.String() != "/items" || sources[0].placementMode != placementModeSequenceItem || sources[1].Destination.String() != "/items/0" || sources[1].placementMode != placementModeMapping { + t.Fatalf("main item sources = %#v", sources) + } + if sources[1].Path != canonicalTestPath(t, helperPath) { + t.Fatalf("helper path = %q", sources[1].Path) + } +} + +func TestItemImportCanJoinAnAuthoredListUnderRegisteredComposition(t *testing.T) { + t.Parallel() + type config struct { + Items []map[string]string `yaml:"items"` + } + directory := t.TempDir() + mainPath := writeImportFile(t, directory, "main.yml", `configkit: + imports: + - path: item.yml + destination: [items] + placement: sequence-item +`) + writeImportFile(t, directory, "item.yml", "configkit: {imports: [{path: helper.yml}]}\nname: imported\n") + writeImportFile(t, directory, "helper.yml", "id: helper-id\n") + input, diagnostics := assembleBaseInput(File(mainPath), Options[config]{Imports: true}) + if len(diagnostics) != 0 { + t.Fatalf("assemble item import: %#v", diagnostics) + } + authored := parseDocumentForDecode(t, "items: [{id: authored-id, name: authored}]\n") + composed, diagnostics := composeDocumentsWithPolicy(sequencePolicy{identityFields: map[string]string{"/items": "id"}}, authored, input.document) + if len(diagnostics) != 0 { + t.Fatalf("compose authored and imported lists: %#v", diagnostics) + } + items, ok := documentAtPath(composed.root, Keys("items")) + if !ok || items.Kind != document.Sequence || len(items.Sequence) != 2 { + t.Fatalf("composed items = %#v", items) + } + if id, ok := documentAtPath(composed.root, Keys("items").Index(1).Key("id")); !ok || id.Text != "helper-id" { + t.Fatalf("imported helper ID = %#v, %v", id, ok) + } + if origin := composed.origins[Keys("items").Index(1).Key("id").String()]; len(origin.Locations) != 1 || origin.Locations[0].SourceID != "source-3" { + t.Fatalf("registered composition origin = %#v", origin) + } +} + func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) { t.Parallel() @@ -201,6 +521,19 @@ func TestParseImportDestinationUsesLiteralStringSegments(t *testing.T) { } } +func TestParseImportPlacementDefaultsPerDeclaration(t *testing.T) { + t.Parallel() + declarations := parseImportDeclarationsYAML(t, `configkit: + imports: + - path: mapping.yml + - glob: records/*.yml + placement: sequence-item +`) + if len(declarations) != 2 || declarations[0].placement != placementModeMapping || declarations[1].placement != placementModeSequenceItem { + t.Fatalf("declaration placements = %#v", declarations) + } +} + func TestParseImportDestinationRejectsInvalidShapesAndMembers(t *testing.T) { t.Parallel() diff --git a/internal/engine/load_input.go b/internal/engine/load_input.go index 8593b40..a29614e 100644 --- a/internal/engine/load_input.go +++ b/internal/engine/load_input.go @@ -28,6 +28,7 @@ type acquiredContribution struct { destination Path relativeDestination Path processing *sourceVersionProcessingResult + nested *composedDocument } // acquisitionScope is the ordered strict mapping scope produced by one input @@ -46,6 +47,7 @@ func cloneAcquisitionScope(scope acquisitionScope) acquisitionScope { source: cloneSourceInfo(contribution.source), payload: contribution.payload, document: contribution.document, destination: contribution.destination.clone(), relativeDestination: contribution.relativeDestination.clone(), processing: contribution.processing, + nested: cloneComposedDocument(contribution.nested), } } } diff --git a/internal/engine/load_request.go b/internal/engine/load_request.go index b4fac8c..7339d1e 100644 --- a/internal/engine/load_request.go +++ b/internal/engine/load_request.go @@ -201,6 +201,12 @@ func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRu composition.inputBytes = accounting.bytes - initialBytes composition.inputNodes = accounting.nodes - initialNodes composition.acquisition = cloneAcquisitionScopes(scopes) + var projectionScope acquisitionScope + for _, scope := range scopes { + projectionScope.contributions = append(projectionScope.contributions, scope.contributions...) + } + projectNestedEvidence(composition, projectionScope) + projectAcquiredDestinations(composition, projectionScope, sources) for pointer, origin := range composition.origins { origin.Layer = LayerUnderlay composition.origins[pointer] = origin diff --git a/internal/engine/native_filesystem_test.go b/internal/engine/native_filesystem_test.go index 57463e5..6df5f8e 100644 --- a/internal/engine/native_filesystem_test.go +++ b/internal/engine/native_filesystem_test.go @@ -42,6 +42,34 @@ items: {main: main} } } +func TestNativeFilesystemItemImportUsesSharedScopedAssembly(t *testing.T) { + t.Parallel() + type item struct { + ID string `yaml:"id"` + Name string `yaml:"name"` + } + type config struct { + Items []item `yaml:"items"` + } + filesystem := fstest.MapFS{ + "main.yml": &fstest.MapFile{Data: []byte("configkit:\n imports:\n - path: records/item.yml\n destination: [items]\n placement: sequence-item\n")}, + "records/item.yml": &fstest.MapFile{Data: []byte("configkit:\n imports:\n - path: helper.yml\nid: native\n")}, + "records/helper.yml": &fstest.MapFile{Data: []byte("name: helper\n")}, + } + namespace := newNativeNamespace("bundle", filesystem) + result, err := testLoad[config](filesystemFile(namespace, "main.yml"), Options[config]{Imports: true}) + if err != nil { + t.Fatalf("native item import: %v", err) + } + if len(result.Config.Items) != 1 || result.Config.Items[0].ID != "native" || result.Config.Items[0].Name != "helper" { + t.Fatalf("native item = %#v", result.Config.Items) + } + sources := result.Sources() + if len(sources) != 3 || sources[1].Destination.String() != "/items" || sources[2].Destination.String() != "/items/0" || sources[1].Path != "records/item.yml" || sources[2].Path != "records/helper.yml" { + t.Fatalf("native item sources = %#v", sources) + } +} + func TestNativeFilesystemNamespaceAndPathFailures(t *testing.T) { first := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte("items: {name: first}\n")}} second := fstest.MapFS{"main.yml": &fstest.MapFile{Data: []byte("items: {name: second}\n")}}