From de28718941daa9423045cc8522ffba2755169844 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 29 Sep 2026 17:13:14 +0000 Subject: [PATCH] Reuse owned assembly across configuration layers --- docs/roadmap/implementation.md | 11 +++ internal/engine/assembly_bench_test.go | 62 ++++++++++++++++ internal/engine/compose.go | 11 +++ internal/engine/explain.go | 3 +- internal/engine/history.go | 30 ++++---- internal/engine/load_input.go | 13 ++-- internal/engine/overlay.go | 99 ++++++++++++++++++-------- internal/engine/overlay_test.go | 41 +++++++++++ internal/engine/override_apply.go | 51 +++++++------ internal/engine/selected_overlay.go | 49 +++++++------ 10 files changed, 278 insertions(+), 92 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 1f242ec..3c39eb2 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -249,6 +249,17 @@ tests and the everyday checker. Benchmarks are evidence, not timing gates. ## Stage 2: Prepare load-owned assembly and incremental provenance +**Completed 2026-09-29.** Ordinary loads now apply selected overlays and CLI +overrides to one owned working document, while copy-preserving helpers remain +available for direct use. Acquired history trees stay detached; composite +contributors survive overlay application. Origin deletion visits only replaced +subtrees, mapping contributors append without copying growing lists, and history +replay shares indexed lookup across applications. Focused behavior tests, the +everyday checker, and the engine race suite passed. On Linux amd64 at 128 small +applications, owned assembly measured about 0.30 ms/op versus 6.0 ms/op for +repeated copy-preserving application; alternating subtree/scalar replacement +measured about 0.39 versus 7.3 ms/op. These benchmarks are evidence, not gates. + Scope: `compose.go`, `overlay.go`, origin indexes, override application, and history replay in `internal/engine`. diff --git a/internal/engine/assembly_bench_test.go b/internal/engine/assembly_bench_test.go index 495fd2e..78908ce 100644 --- a/internal/engine/assembly_bench_test.go +++ b/internal/engine/assembly_bench_test.go @@ -42,6 +42,68 @@ func BenchmarkOverlayRecursiveMappings(b *testing.B) { } } +func BenchmarkOverlayManyApplications(b *testing.B) { + for _, count := range []int{32, 64, 128} { + inputs := make([]*document.Node, count) + for index := range inputs { + inputs[index] = benchmarkCatalog("source-2", index, 1) + } + benchmarkOverlayApplications(b, count, inputs) + } +} + +func BenchmarkOverlayRepeatedReplacement(b *testing.B) { + for _, count := range []int{32, 64, 128} { + inputs := make([]*document.Node, count) + for index := range inputs { + location := document.Location{SourceID: "source-2"} + value := &document.Node{Kind: document.String, Text: "replacement", Location: location} + if index%2 == 0 { + value = &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{{ + Key: "enabled", Value: &document.Node{Kind: document.Boolean, Text: "false", Location: location}, + }}} + } + inputs[index] = &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{{ + Key: "catalog", Value: &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{{ + Key: "item-0", Value: value, + }}}, + }}} + } + benchmarkOverlayApplications(b, count, inputs) + } +} + +func benchmarkOverlayApplications(b *testing.B, count int, inputs []*document.Node) { + base, diagnostics := composeDocuments(benchmarkCatalog("source-1", 0, 1)) + if len(diagnostics) != 0 { + b.Fatal(diagnostics) + } + for _, mode := range []string{"owned", "copy-preserving"} { + b.Run(mode+"/"+strconv.Itoa(count), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if mode == "owned" { + assembly := newAssemblyAccumulator(cloneComposedDocument(base)) + for _, input := range inputs { + if diagnostics := assembly.applyOverlay(input, nil, nil, LayerOverlay); len(diagnostics) != 0 { + b.Fatal(diagnostics) + } + } + continue + } + current := base + for _, input := range inputs { + current, diagnostics = overlayDocuments(current, input) + if len(diagnostics) != 0 { + b.Fatal(diagnostics) + } + } + } + }) + } +} + func benchmarkCatalog(source string, first, width int) *document.Node { location := document.Location{SourceID: source} entries := make([]document.Entry, width) diff --git a/internal/engine/compose.go b/internal/engine/compose.go index fa9db79..5ef008c 100644 --- a/internal/engine/compose.go +++ b/internal/engine/compose.go @@ -15,6 +15,17 @@ type composedDocument struct { inputNodes int } +// assemblyAccumulator owns one mutable working document. Acquired trees held by +// history applications are separate from its root and remain unchanged. +type assemblyAccumulator struct { + current *composedDocument + index mappingIndex +} + +func newAssemblyAccumulator(owned *composedDocument) *assemblyAccumulator { + return &assemblyAccumulator{current: owned} +} + // cloneComposedDocument detaches mutable composition and provenance state; // history application trees are immutable and retain their existing sharing. func cloneComposedDocument(base *composedDocument) *composedDocument { diff --git a/internal/engine/explain.go b/internal/engine/explain.go index c6b6a2a..07c6411 100644 --- a/internal/engine/explain.go +++ b/internal/engine/explain.go @@ -51,13 +51,14 @@ func (r *Inspection) Explain(path Path) (Explanation, bool) { After: valueSnapshot(working, path, r.secrets), Origin: Origin{Layer: LayerDefault}, }) } + replayIndex := new(mappingIndex) for _, application := range r.applications { operationPath, relevant := applicationOperationPath(application, path) var before ValueSnapshot if relevant { before = valueSnapshot(working, path, r.secrets) } - applyHistoryApplication(&working, application) + applyHistoryApplication(&working, application, replayIndex) if !relevant { continue } diff --git a/internal/engine/history.go b/internal/engine/history.go index 4f42892..f0aa152 100644 --- a/internal/engine/history.go +++ b/internal/engine/history.go @@ -83,15 +83,15 @@ func pathsIntersect(left, right Path) bool { return pathContains(left, right) || pathContains(right, left) } -func applyHistoryApplication(root **document.Node, application historyApplication) { +func applyHistoryApplication(root **document.Node, application historyApplication, index *mappingIndex) { if application.path.String() == "" { - mergeHistoryDocument(root, application.value) + mergeHistoryDocument(root, application.value, index) return } - mergeHistoryAtPath(root, application.path, application.value) + mergeHistoryAtPath(root, application.path, application.value, index) } -func mergeHistoryDocument(target **document.Node, incoming *document.Node) { +func mergeHistoryDocument(target **document.Node, incoming *document.Node, index *mappingIndex) { if incoming == nil { return } @@ -100,39 +100,39 @@ func mergeHistoryDocument(target **document.Node, incoming *document.Node) { return } for _, entry := range incoming.Mapping { - position := documentEntryIndex((*target).Mapping, entry.Key) + position := index.find(*target, entry.Key) if position < 0 { cloned := entry cloned.Value = document.Clone(entry.Value) - (*target).Mapping = append((*target).Mapping, cloned) + index.append(*target, cloned) continue } - mergeHistoryDocument(&(*target).Mapping[position].Value, entry.Value) + mergeHistoryDocument(&(*target).Mapping[position].Value, entry.Value, index) } } -func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node) { +func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node, index *mappingIndex) { if root == nil || *root == nil || len(path.segments) == 0 { - mergeHistoryDocument(root, incoming) + mergeHistoryDocument(root, incoming, index) return } current := *root - for index, segment := range path.segments { + for segmentIndex, segment := range path.segments { if current.Kind != document.Mapping || segment.kind == pathSegmentIndex { return } - position := documentEntryIndex(current.Mapping, segment.key) - if index == len(path.segments)-1 { + position := index.find(current, segment.key) + if segmentIndex == len(path.segments)-1 { if position < 0 { - current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, Value: document.Clone(incoming)}) + index.append(current, document.Entry{Key: segment.key, Value: document.Clone(incoming)}) } else { - mergeHistoryDocument(¤t.Mapping[position].Value, incoming) + mergeHistoryDocument(¤t.Mapping[position].Value, incoming, index) } return } if position < 0 { child := &document.Node{Kind: document.Mapping} - current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, Value: child}) + index.append(current, document.Entry{Key: segment.key, Value: child}) current = child } else { current = current.Mapping[position].Value diff --git a/internal/engine/load_input.go b/internal/engine/load_input.go index 4b6420f..bc5554a 100644 --- a/internal/engine/load_input.go +++ b/internal/engine/load_input.go @@ -47,9 +47,10 @@ func applyInputLayers[T any](source Source, options Options[T], input assembledI if len(diagnostics) != 0 { return input, diagnostics } + assembly := newAssemblyAccumulator(composition) if activeOverlay(options.Overlay) { - composition, input.sources, diagnostics = loadSelectedOverlayDocument( - source, options.Filesystem, input.parsed, composition, input.sources, + input.sources, diagnostics = loadSelectedOverlayInto( + source, options.Filesystem, input.parsed, assembly, input.sources, options.Overlay, options.Definition.Version, defaultImportLimits(), ) if len(diagnostics) != 0 { @@ -57,15 +58,15 @@ func applyInputLayers[T any](source Source, options Options[T], input assembledI } } if len(options.Overrides) != 0 { - composition, input.sources, diagnostics = applyOverrides( - composition, input.sources, options.Overrides, options.Definition.Version, defaultImportLimits(), + input.sources, diagnostics = assembly.applyOverrides( + input.sources, options.Overrides, options.Definition.Version, defaultImportLimits(), ) if len(diagnostics) != 0 { return input, diagnostics } } - input.composition = composition - input.document = composition.root + input.composition = assembly.current + input.document = assembly.current.root return input, nil } diff --git a/internal/engine/overlay.go b/internal/engine/overlay.go index 6f86e0b..55280e0 100644 --- a/internal/engine/overlay.go +++ b/internal/engine/overlay.go @@ -18,24 +18,41 @@ type displacedDefinition struct { } func overlayDocuments(base *composedDocument, overlay *document.Node) (*composedDocument, []Diagnostic) { - if base == nil || base.root == nil || base.root.Kind != document.Mapping || overlay == nil || overlay.Kind != document.Mapping { - return nil, []Diagnostic{{ - Code: CodeOverlayInvalid, Severity: SeverityError, - Message: "overlay requires mapping document roots", - }} + if base == nil { + return nil, invalidOverlayDocument() } - result := cloneComposedDocument(base) - mergeOverlayOrigin(result.origins, Path{}, overlay.Location, LayerOverlay) - mergeOverlayMappings(result, result.root, overlay, Path{}, LayerOverlay, true) - result.applications = append(result.applications, historyApplication{layer: LayerOverlay, value: overlay}) - return result, nil + assembly := newAssemblyAccumulator(cloneComposedDocument(base)) + if diagnostics := assembly.applyOverlay(overlay, nil, nil, LayerOverlay); len(diagnostics) != 0 { + return nil, diagnostics + } + return assembly.current, nil } -func mergeOverlayMappings(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool) { - mergeOverlayMappingsIndexed(result, target, overlay, current, layer, recordReplacements, new(mappingIndex)) +func invalidOverlayDocument() []Diagnostic { + return []Diagnostic{{ + Code: CodeOverlayInvalid, Severity: SeverityError, + Message: "overlay requires mapping document roots", + }} } -func mergeOverlayMappingsIndexed(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool, index *mappingIndex) { +func (a *assemblyAccumulator) applyOverlay(overlay *document.Node, incomingOrigins map[string]Origin, applications []historyApplication, layer Layer) []Diagnostic { + if a == nil || a.current == nil || a.current.root == nil || a.current.root.Kind != document.Mapping || overlay == nil || overlay.Kind != document.Mapping { + return invalidOverlayDocument() + } + mergeOriginContribution(a.current.origins, Path{}, incomingOrigin(incomingOrigins, Path{}, overlay, layer)) + mergeOverlayMappings(a.current, a.current.root, overlay, Path{}, layer, true, incomingOrigins, &a.index) + if len(applications) == 0 { + a.current.applications = append(a.current.applications, historyApplication{layer: layer, value: overlay}) + } else { + for _, application := range applications { + application.layer = layer + a.current.applications = append(a.current.applications, application) + } + } + return nil +} + +func mergeOverlayMappings(result *composedDocument, target, overlay *document.Node, current Path, layer Layer, recordReplacements bool, incomingOrigins map[string]Origin, index *mappingIndex) { for _, incoming := range overlay.Mapping { path := current.Key(incoming.Key) position := index.find(target, incoming.Key) @@ -43,27 +60,27 @@ func mergeOverlayMappingsIndexed(result *composedDocument, target, overlay *docu cloned := incoming cloned.Value = document.Clone(incoming.Value) index.append(target, cloned) - collectOverlayOrigins(result.origins, incoming.Value, path, layer) + collectOverlayOriginsFrom(result.origins, incomingOrigins, incoming.Value, path, layer) continue } existing := target.Mapping[position].Value if existing.Kind == document.Mapping && incoming.Value.Kind == document.Mapping { - mergeOverlayOrigin(result.origins, path, incoming.Value.Location, layer) - mergeOverlayMappingsIndexed(result, existing, incoming.Value, path, layer, recordReplacements, index) + mergeOriginContribution(result.origins, path, incomingOrigin(incomingOrigins, path, incoming.Value, layer)) + mergeOverlayMappings(result, existing, incoming.Value, path, layer, recordReplacements, incomingOrigins, index) continue } displaced := displacedOrigins(existing, result.origins, path) previous := cloneOrigin(result.origins[path.String()]) - deleteOriginSubtree(result.origins, path) + deleteOriginSubtree(result.origins, path, existing) target.Mapping[position] = incoming target.Mapping[position].Value = document.Clone(incoming.Value) - collectOverlayOrigins(result.origins, incoming.Value, path, layer) + collectOverlayOriginsFrom(result.origins, incomingOrigins, incoming.Value, path, layer) if recordReplacements { result.overlays = append(result.overlays, overlayRecord{ change: Change{ - Path: path.clone(), Origin: overlayOrigin(layer, incoming.Value.Location), + Path: path.clone(), Origin: incomingOrigin(incomingOrigins, path, incoming.Value, layer), Previous: previousOrigins(previous), }, displaced: displaced, @@ -73,27 +90,44 @@ func mergeOverlayMappingsIndexed(result *composedDocument, target, overlay *docu } func collectOverlayOrigins(origins map[string]Origin, node *document.Node, path Path, layer Layer) { - origins[path.String()] = overlayOrigin(layer, node.Location) + collectOverlayOriginsFrom(origins, nil, node, path, layer) +} + +func collectOverlayOriginsFrom(origins, incomingOrigins map[string]Origin, node *document.Node, path Path, layer Layer) { + origins[path.String()] = incomingOrigin(incomingOrigins, path, node, layer) switch node.Kind { case document.Mapping: for _, entry := range node.Mapping { - collectOverlayOrigins(origins, entry.Value, path.Key(entry.Key), layer) + collectOverlayOriginsFrom(origins, incomingOrigins, entry.Value, path.Key(entry.Key), layer) } case document.Sequence: for index, child := range node.Sequence { - collectOverlayOrigins(origins, child, path.Index(index), layer) + collectOverlayOriginsFrom(origins, incomingOrigins, child, path.Index(index), layer) } } } func mergeOverlayOrigin(origins map[string]Origin, path Path, location document.Location, layer Layer) { + mergeOriginContribution(origins, path, overlayOrigin(layer, location)) +} + +func mergeOriginContribution(origins map[string]Origin, path Path, incoming Origin) { pointer := path.String() - origin := cloneOrigin(origins[pointer]) - origin.Layer = layer - origin.Locations = append(origin.Locations, publicLocation(location)) + origin := origins[pointer] + origin.Layer = incoming.Layer + origin.Locations = append(origin.Locations, incoming.Locations...) origins[pointer] = origin } +func incomingOrigin(origins map[string]Origin, path Path, node *document.Node, layer Layer) Origin { + if origin, ok := origins[path.String()]; ok { + origin = cloneOrigin(origin) + origin.Layer = layer + return origin + } + return overlayOrigin(layer, node.Location) +} + func overlayOrigin(layer Layer, location document.Location) Origin { return Origin{Layer: layer, Locations: []Location{publicLocation(location)}} } @@ -133,11 +167,16 @@ func collectDisplacedOrigins(displaced *[]displacedDefinition, node *document.No } } -func deleteOriginSubtree(origins map[string]Origin, path Path) { - prefix := path.String() - for pointer := range origins { - if pointer == prefix || strings.HasPrefix(pointer, prefix+"/") { - delete(origins, pointer) +func deleteOriginSubtree(origins map[string]Origin, path Path, node *document.Node) { + delete(origins, path.String()) + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + deleteOriginSubtree(origins, path.Key(entry.Key), entry.Value) + } + case document.Sequence: + for index, child := range node.Sequence { + deleteOriginSubtree(origins, path.Index(index), child) } } } diff --git a/internal/engine/overlay_test.go b/internal/engine/overlay_test.go index 3b64191..6b052be 100644 --- a/internal/engine/overlay_test.go +++ b/internal/engine/overlay_test.go @@ -76,3 +76,44 @@ to_scalar: scalar t.Fatal("overlay result aliases an input document") } } + +func TestOwnedOverlayKeepsCompositeContributorsAndAcquiredInputs(t *testing.T) { + baseInput := parseCompositionDocument(t, "source-1", "catalog:\n base: true\n") + base, diagnostics := composeDocuments(baseInput) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + first := parseCompositionDocument(t, "source-2", "catalog:\n first: true\n") + second := parseCompositionDocument(t, "source-3", "catalog:\n second: true\n") + firstBefore, secondBefore := document.Clone(first), document.Clone(second) + incoming, diagnostics := composeDocuments(first, second) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + assembly := newAssemblyAccumulator(base) + if diagnostics := assembly.applyOverlay(incoming.root, incoming.origins, incoming.applications, LayerOverlay); len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + for _, key := range []string{"base", "first", "second"} { + if _, ok := documentAtPath(assembly.current.root, Keys("catalog", key)); !ok { + t.Fatalf("catalog is missing %q", key) + } + } + origin := assembly.current.origins[Keys("catalog").String()] + if got, want := origin.Locations, []Location{{SourceID: "source-1", Line: 2, Column: 3}, {SourceID: "source-2", Line: 2, Column: 3}, {SourceID: "source-3", Line: 2, Column: 3}}; !reflect.DeepEqual(got, want) { + t.Fatalf("catalog origin locations = %#v, want %#v", got, want) + } + if len(assembly.current.applications) != 3 { + t.Fatalf("history applications = %d, want three source contributions", len(assembly.current.applications)) + } + for _, application := range assembly.current.applications[1:] { + if application.layer != LayerOverlay { + t.Fatalf("incoming application layer = %q", application.layer) + } + } + assembly.current.root.Mapping[0].Value.Mapping[1].Value.Text = "false" + if !reflect.DeepEqual(first, firstBefore) || !reflect.DeepEqual(second, secondBefore) || + !reflect.DeepEqual(first, assembly.current.applications[1].value) || !reflect.DeepEqual(second, assembly.current.applications[2].value) { + t.Fatal("working tree mutation changed an acquired input") + } +} diff --git a/internal/engine/override_apply.go b/internal/engine/override_apply.go index 0add545..64b5c19 100644 --- a/internal/engine/override_apply.go +++ b/internal/engine/override_apply.go @@ -11,31 +11,42 @@ func applyOverrides(base *composedDocument, sources []SourceInfo, overrides []Ov if len(overrides) == 0 { return base, cloneSourceInfos(sources), nil } - prepared, allSources, diagnostics := prepareOverrides(base, sources, overrides, rule, limits) + assembly := newAssemblyAccumulator(cloneComposedDocument(base)) + allSources, diagnostics := assembly.applyOverrides(sources, overrides, rule, limits) if len(diagnostics) != 0 { return nil, allSources, diagnostics } - result := cloneComposedDocument(base) - for _, override := range prepared { - applyPreparedOverride(result, override) - result.inputBytes += override.bytes - result.inputNodes += override.nodes - } - return result, allSources, nil + return assembly.current, allSources, nil } -func applyPreparedOverride(result *composedDocument, override preparedOverride) { +func (a *assemblyAccumulator) applyOverrides(sources []SourceInfo, overrides []Override, rule *VersionRule, limits importLimits) ([]SourceInfo, []Diagnostic) { + if len(overrides) == 0 { + return cloneSourceInfos(sources), nil + } + prepared, allSources, diagnostics := prepareOverrides(a.current, sources, overrides, rule, limits) + if len(diagnostics) != 0 { + return allSources, diagnostics + } + for _, override := range prepared { + applyPreparedOverride(a.current, override, &a.index) + a.current.inputBytes += override.bytes + a.current.inputNodes += override.nodes + } + return allSources, nil +} + +func applyPreparedOverride(result *composedDocument, override preparedOverride, index *mappingIndex) { location := document.Location{SourceID: override.source.ID} mergeOverlayOrigin(result.origins, Path{}, location, LayerCLI) current := result.root currentPath := Path{} - for index, segment := range override.path.segments { + for segmentIndex, segment := range override.path.segments { path := currentPath.Key(segment.key) - position := documentEntryIndex(current.Mapping, segment.key) - if index < len(override.path.segments)-1 { + position := index.find(current, segment.key) + if segmentIndex < len(override.path.segments)-1 { if position < 0 { child := &document.Node{Kind: document.Mapping, Location: location, Mapping: []document.Entry{}} - current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: child}) + index.append(current, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: child}) collectOverlayOrigins(result.origins, child, path, LayerCLI) current = child } else { @@ -50,16 +61,16 @@ func applyPreparedOverride(result *composedDocument, override preparedOverride) var displaced []displacedDefinition if position < 0 { incoming := document.Clone(override.value) - current.Mapping = append(current.Mapping, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: incoming}) + index.append(current, document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: incoming}) collectOverlayOrigins(result.origins, incoming, path, LayerCLI) } else { existing := current.Mapping[position].Value - displaced = affectedOverrideOrigins(existing, override.value, result.origins, path) + displaced = affectedOverrideOrigins(existing, override.value, result.origins, path, index) if existing.Kind == document.Mapping && override.value.Kind == document.Mapping { mergeOverlayOrigin(result.origins, path, location, LayerCLI) - mergeOverlayMappings(result, existing, override.value, path, LayerCLI, false) + mergeOverlayMappings(result, existing, override.value, path, LayerCLI, false, nil, index) } else { - deleteOriginSubtree(result.origins, path) + deleteOriginSubtree(result.origins, path, existing) current.Mapping[position] = document.Entry{Key: segment.key, KeyLocation: location, ValueLocation: location, Value: document.Clone(override.value)} collectOverlayOrigins(result.origins, current.Mapping[position].Value, path, LayerCLI) } @@ -74,18 +85,18 @@ func applyPreparedOverride(result *composedDocument, override preparedOverride) } } -func affectedOverrideOrigins(existing, incoming *document.Node, origins map[string]Origin, path Path) []displacedDefinition { +func affectedOverrideOrigins(existing, incoming *document.Node, origins map[string]Origin, path Path, index *mappingIndex) []displacedDefinition { displaced := []displacedDefinition{{path: path.clone(), origin: cloneOrigin(origins[path.String()])}} if existing.Kind == document.Mapping && incoming.Kind == document.Mapping { for _, entry := range incoming.Mapping { - position := documentEntryIndex(existing.Mapping, entry.Key) + position := index.find(existing, entry.Key) if position < 0 { continue } childPath := path.Key(entry.Key) childExisting := existing.Mapping[position].Value if childExisting.Kind == document.Mapping && entry.Value.Kind == document.Mapping { - displaced = append(displaced, affectedOverrideOrigins(childExisting, entry.Value, origins, childPath)...) + displaced = append(displaced, affectedOverrideOrigins(childExisting, entry.Value, origins, childPath, index)...) } else { displaced = append(displaced, displacedOrigins(childExisting, origins, childPath)...) } diff --git a/internal/engine/selected_overlay.go b/internal/engine/selected_overlay.go index ad565ee..a435a2e 100644 --- a/internal/engine/selected_overlay.go +++ b/internal/engine/selected_overlay.go @@ -32,9 +32,18 @@ func validOverlayName(name string) bool { } func loadSelectedOverlayDocument(source Source, context *FilesystemContext, main parsedSource, base *composedDocument, sources []SourceInfo, name string, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) { + assembly := newAssemblyAccumulator(cloneComposedDocument(base)) + allSources, diagnostics := loadSelectedOverlayInto(source, context, main, assembly, sources, name, rule, limits) + if len(diagnostics) != 0 { + return nil, allSources, diagnostics + } + return assembly.current, allSources, nil +} + +func loadSelectedOverlayInto(source Source, context *FilesystemContext, main parsedSource, base *assemblyAccumulator, sources []SourceInfo, name string, rule *VersionRule, limits importLimits) ([]SourceInfo, []Diagnostic) { rootPath, _, _, diagnostic := importFilesystem(source, context, main) if diagnostic != nil { - return nil, cloneSourceInfos(sources), []Diagnostic{*diagnostic} + return cloneSourceInfos(sources), []Diagnostic{*diagnostic} } overlayPath := filepath.Join(rootPath, "overlays", name+".yml") canonical, err := canonicalExistingPath(overlayPath) @@ -46,25 +55,25 @@ func loadSelectedOverlayDocument(source Source, context *FilesystemContext, main } else if errors.Is(err, fs.ErrPermission) { code, message = CodeImportPermission, "selected overlay cannot be accessed" } - return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(code, message, nil)} + return cloneSourceInfos(sources), []Diagnostic{importDiagnostic(code, message, nil)} } if !containedPath(rootPath, canonical) { - return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeOverlayOutsideRoot, "selected overlay resolves outside the configuration root", nil)} + return cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeOverlayOutsideRoot, "selected overlay resolves outside the configuration root", nil)} } for _, existing := range sources { if existing.Path != "" && existing.Path == canonical { - return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeOverlayRepeated, "selected overlay is already present in the base source graph", nil)} + return cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeOverlayRepeated, "selected overlay is already present in the base source graph", nil)} } } if len(sources)+1 > limits.sources { - return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeOverlayLimitExceeded, "selected overlay exceeds the source-count limit", nil)} + return cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeOverlayLimitExceeded, "selected overlay exceeds the source-count limit", nil)} } root, err := os.OpenRoot(rootPath) if err != nil { value := globFilesystemDiagnostic(err, "configuration root could not be opened for the selected overlay", Location{}) value.Primary = nil - return nil, cloneSourceInfos(sources), []Diagnostic{value} + return cloneSourceInfos(sources), []Diagnostic{value} } rootClosed := false defer func() { @@ -74,31 +83,31 @@ func loadSelectedOverlayDocument(source Source, context *FilesystemContext, main }() relative, err := filepath.Rel(rootPath, canonical) if err != nil || filepath.IsAbs(relative) || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return nil, cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeOverlayOutsideRoot, "selected overlay could not be addressed inside the configuration root", nil)} + return cloneSourceInfos(sources), []Diagnostic{importDiagnostic(CodeOverlayOutsideRoot, "selected overlay could not be addressed inside the configuration root", nil)} } sourceID := "source-" + strconv.Itoa(len(sources)+1) overlayLocation := &Location{SourceID: sourceID} overlaySource := SourceInfo{ID: sourceID, Label: filepath.ToSlash(filepath.Join("overlays", name+".yml")), Path: canonical, Layer: LayerOverlay} sources = append(cloneSourceInfos(sources), overlaySource) - data, diagnostic := readOverlayFile(root, relative, sourceID, limits.bytes-base.inputBytes) + data, diagnostic := readOverlayFile(root, relative, sourceID, limits.bytes-base.current.inputBytes) if diagnostic != nil { - return nil, sources, []Diagnostic{*diagnostic} + return sources, []Diagnostic{*diagnostic} } overlayRoot, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{Depth: maxDocumentDepth, Nodes: maxDocumentNodes}) if len(issues) != 0 { - return nil, sources, parserDiagnostics(sourceID, issues) + return sources, parserDiagnostics(sourceID, issues) } nodeCount := countInputNodes(overlayRoot) - if base.inputNodes+nodeCount > limits.nodes { - return nil, sources, []Diagnostic{importDiagnostic(CodeOverlayLimitExceeded, "selected overlay exceeds the aggregate node limit", overlayLocation)} + if base.current.inputNodes+nodeCount > limits.nodes { + return sources, []Diagnostic{importDiagnostic(CodeOverlayLimitExceeded, "selected overlay exceeds the aggregate node limit", overlayLocation)} } if diagnostics := overlayDirectiveDiagnostics(overlayRoot, Path{}); len(diagnostics) != 0 { - return nil, sources, diagnostics + return sources, diagnostics } if versionNode := fragmentVersionClaim(overlayRoot, rule); versionNode != nil { location := publicLocation(versionNode.Location) - return nil, sources, []Diagnostic{{ + return sources, []Diagnostic{{ Code: CodeOverlayVersion, Severity: SeverityError, Path: rule.Path.clone(), Message: "overlays cannot contribute to or replace the schema-version path", Primary: &location, }} @@ -106,15 +115,15 @@ func loadSelectedOverlayDocument(source Source, context *FilesystemContext, main closeErr := root.Close() rootClosed = true if closeErr != nil { - return nil, sources, []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed after reading the selected overlay", nil)} + return sources, []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed after reading the selected overlay", nil)} } - overlayd, diagnostics := overlayDocuments(base, overlayRoot) - if overlayd != nil { - overlayd.inputBytes = base.inputBytes + int64(len(data)) - overlayd.inputNodes = base.inputNodes + nodeCount + if diagnostics := base.applyOverlay(overlayRoot, nil, nil, LayerOverlay); len(diagnostics) != 0 { + return sources, diagnostics } - return overlayd, sources, diagnostics + base.current.inputBytes += int64(len(data)) + base.current.inputNodes += nodeCount + return sources, nil } func readOverlayFile(root *os.Root, relative, sourceID string, remaining int64) ([]byte, *Diagnostic) {