Integrate private destinations for layered inputs

docs/roadmap/implementation.md

@@ -203,7 +203,7 @@ No loader accepts a new public field yet.
 
 ## Stage 3 — Integrate private non-import input placement
 
-Status: planned. Scope: engine request plumbing and ordinary acquired bodies.
+Status: complete. Scope: engine request plumbing and ordinary acquired bodies.
 
 1. Add destination fields to engine `Input`, `loadInput`, and `SourceInfo`.
    Preserve ownership through engine `LoadLayered` request construction,
@@ -221,9 +221,10 @@ Status: planned. Scope: engine request plumbing and ordinary acquired bodies.
    rejection without consuming a test-owned reader. Reuse current source-kind
    acquisition tests; full source equivalence is verified publicly in Stage 10.
 
-Validate: destination, grouped-load, source, and existing ordinary-load tests;
+Validated with `go test ./internal/engine -run 'Destination|PlaceDocument|Grouped'`
-the everyday check. Done: private non-import placement reaches assembly, with
+and `./scripts/check.sh` (go1.26.8 linux/amd64); both passed. Private non-import
-root behavior unchanged. Imports are the next private integration step.
+placement reaches assembly, with root behavior unchanged. Imports are the next
+private integration step.
 
 ## Stage 4 — Carry placement through import acquisition
 

internal/engine/destination.go

@@ -29,6 +29,20 @@ func normalizeDestination(path Path) (Path, error) {
 	return Path{segments: segments}, nil
 }
 
+func invalidDestinationDiagnostic(destination Path) Diagnostic {
+	return Diagnostic{
+		Code: CodeOptionsInvalid, Severity: SeverityError,
+		Path:    destination.clone(),
+		Message: "input destination must contain only non-reserved mapping keys",
+	}
+}
+
+func setSourceDestinations(sources []SourceInfo, destination Path) {
+	for index := range sources {
+		sources[index].Destination = destination.clone()
+	}
+}
+
 // placeDocument checks all per-document bounds before allocating destination
 // wrappers. Aggregate accounting remains the caller's responsibility.
 func placeDocument(body *document.Node, destination Path, sourceRoot document.Location, maxDepth, maxNodes int) (Path, *document.Node, error) {

internal/engine/destination_load_test.go

@@ -0,0 +1,118 @@
+package engine
+
+import (
+	"bytes"
+	"testing"
+)
+
+type destinationLoadConfig struct {
+	Main        map[string]string            `yaml:"main"`
+	Defaults    map[string]string            `yaml:"defaults"`
+	Collections map[string]map[string]string `yaml:"collections"`
+	Value       string                       `yaml:"value"`
+}
+
+func TestLayeredLoadPlacesNonImportMainAndUnderlayInputs(t *testing.T) {
+	t.Parallel()
+
+	destination := Keys("main")
+	main := Input{Source: Bytes("main", []byte("value: primary\n")), Destination: destination}
+	underlay := Input{Source: Bytes("defaults", []byte("value: fallback\n")), Destination: Keys("defaults")}
+	loaded, err := LoadLayered(main, testSecretBinding(), LayeredOptions[destinationLoadConfig]{
+		Underlays: []Underlay{{Inputs: []Input{underlay}}},
+	})
+	if err != nil {
+		t.Fatalf("LoadLayered: %v", err)
+	}
+	result := testLoaded(loaded)
+	if result.Config.Main["value"] != "primary" || result.Config.Defaults["value"] != "fallback" {
+		t.Fatalf("mounted configuration = %#v", result.Config)
+	}
+	sources := result.Sources()
+	if len(sources) != 2 || sources[0].Destination.String() != "/main" || sources[1].Destination.String() != "/defaults" {
+		t.Fatalf("source destinations = %#v", sources)
+	}
+
+	destination.segments[0].key = "caller mutation"
+	sources[0].Destination.segments[0].key = "record mutation"
+	if got := result.Sources()[0].Destination.String(); got != "/main" {
+		t.Fatalf("source destination was not independently owned: %q", got)
+	}
+}
+
+func TestLayeredLoadOmittedAndEmptyDestinationsKeepRootPlacement(t *testing.T) {
+	t.Parallel()
+
+	inputs := []Input{
+		{Source: Bytes("omitted", []byte("value: same\n"))},
+		{Source: Bytes("empty", []byte("value: same\n")), Destination: Keys()},
+	}
+	for _, input := range inputs {
+		loaded, err := LoadLayered(input, testSecretBinding(), LayeredOptions[destinationLoadConfig]{})
+		if err != nil {
+			t.Fatalf("LoadLayered: %v", err)
+		}
+		result := testLoaded(loaded)
+		if result.Config.Value != "same" {
+			t.Fatalf("root value = %q", result.Config.Value)
+		}
+		sources := result.Sources()
+		if len(sources) != 1 || sources[0].Destination.String() != "" {
+			t.Fatalf("root source destination = %#v", sources)
+		}
+	}
+}
+
+func TestLayeredLoadDistinguishesEmptyBodyFromNoSource(t *testing.T) {
+	t.Parallel()
+
+	loaded, err := LoadLayered(
+		Input{Source: NoSource(), Destination: Keys("absent")},
+		testSecretBinding(),
+		LayeredOptions[destinationLoadConfig]{Underlays: []Underlay{{Inputs: []Input{{
+			Source: Bytes("empty", []byte("{}\n")), Destination: Keys("collections", "one"),
+		}}}}},
+	)
+	if err != nil {
+		t.Fatalf("LoadLayered: %v", err)
+	}
+	result := testLoaded(loaded)
+	info, ok := result.Metadata().Lookup(Keys("collections", "one"))
+	if !ok || !info.Explicit || info.Origin.Layer != LayerUnderlay {
+		t.Fatalf("empty mounted mapping = %#v, %v", info, ok)
+	}
+	if _, ok := result.Metadata().Lookup(Keys("absent")); ok {
+		t.Fatal("NoSource materialized its destination")
+	}
+	sources := result.Sources()
+	if len(sources) != 1 || sources[0].Label != "empty" || sources[0].Destination.String() != "/collections/one" {
+		t.Fatalf("sources = %#v", sources)
+	}
+}
+
+func TestLayeredLoadPreflightsDestinationsBeforeReadingSources(t *testing.T) {
+	t.Parallel()
+
+	reader := bytes.NewBufferString("value: unused\n")
+	_, err := LoadLayered(
+		Input{Source: NoSource()},
+		testSecretBinding(),
+		LayeredOptions[destinationLoadConfig]{Underlays: []Underlay{{Inputs: []Input{{
+			Source: Reader("reader", reader), Destination: Keys("configkit"),
+		}}}}},
+	)
+	if err == nil {
+		t.Fatal("reserved destination was accepted")
+	}
+	if reader.Len() == 0 {
+		t.Fatal("source reader was consumed before destination preflight")
+	}
+	loadError, ok := err.(*LoadError)
+	if !ok || len(loadError.Diagnostics()) != 1 {
+		t.Fatalf("error = %#v", err)
+	}
+	diagnostic := loadError.Diagnostics()[0]
+	if diagnostic.Code != CodeOptionsInvalid || diagnostic.Path.String() != "/configkit" || diagnostic.Primary != nil {
+		t.Fatalf("destination diagnostic = %#v", diagnostic)
+	}
+}

internal/engine/layered.go

@@ -2,11 +2,12 @@ package engine
 
 import "os"
 
-// Input selects one source and its independent import policy.
+// Input selects one source, its destination, and its independent import policy.
 type Input struct {
-	Source     Source
+	Source      Source
-	Imports    bool
+	Imports     bool
-	Filesystem *FilesystemContext
+	Filesystem  *FilesystemContext
+	Destination Path
 }
 
 // Underlay contains strictly composed contributions at one precedence tier.
@@ -32,7 +33,10 @@ func LoadLayered[T any](main Input, secrets SecretBinding, supplied ...LayeredOp
 		options = supplied[0]
 	}
 	request := loadRequest[T]{
-		main:       loadInput{source: main.Source, imports: main.Imports, filesystem: main.Filesystem},
+		main: loadInput{
+			source: main.Source, imports: main.Imports, filesystem: main.Filesystem,
+			destination: main.Destination.clone(),
+		},
 		definition: options.Definition, overlay: options.Overlay,
 		overlayFilesystem: options.OverlayFilesystem,
 		overrides:         append([]Override(nil), options.Overrides...),
@@ -43,6 +47,7 @@ func LoadLayered[T any](main Input, secrets SecretBinding, supplied ...LayeredOp
 		for inputIndex, input := range underlay.Inputs {
 			request.underlays[index].inputs[inputIndex] = loadInput{
 				source: input.Source, imports: input.Imports, filesystem: input.Filesystem,
+				destination: input.Destination.clone(),
 			}
 		}
 	}

internal/engine/load.go

@@ -155,7 +155,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
 		mainOptions.Overlay = request.overlay
 	}
 
-	input, diagnostics := assembleBaseInputTracked(request.main.source, mainOptions, accounting, nil, nil)
+	input, diagnostics := assembleBaseInputTracked(request.main.source, mainOptions, request.main.destination, accounting, nil, nil)
 	accumulator.sources = input.sources
 	accumulator.add(diagnostics...)
 	if len(diagnostics) != 0 {

internal/engine/load_input.go

@@ -17,10 +17,10 @@ type assembledInput struct {
 }
 
 func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput, []Diagnostic) {
-	return assembleBaseInputTracked(source, options, &importAccounting{}, nil, nil)
+	return assembleBaseInputTracked(source, options, Path{}, &importAccounting{}, nil, nil)
 }
 
-func assembleBaseInputTracked[T any](source Source, options Options[T], accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (assembledInput, []Diagnostic) {
+func assembleBaseInputTracked[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (assembledInput, []Diagnostic) {
 	limits := defaultImportLimits()
 	parseLimits := defaultParseLimits()
 	parseLimits.source.remaining = limits.bytes - accounting.bytes
@@ -28,6 +28,7 @@ func assembleBaseInputTracked[T any](source Source, options Options[T], accounti
 	input := assembledInput{parsed: parsed, document: parsed.document}
 	if len(diagnostics) != 0 {
 		input.sources = sourceInfos(parsed.source)
+		setSourceDestinations(input.sources, destination)
 		return input, diagnostics
 	}
 	if options.Imports {
@@ -38,6 +39,7 @@ func assembleBaseInputTracked[T any](source Source, options Options[T], accounti
 		input.document = input.composition.root
 	} else {
 		input.sources = sourceInfos(parsed.source)
+		setSourceDestinations(input.sources, destination)
 		if options.Filesystem != nil && !activeOverlay(options.Overlay) {
 			if diagnostic := validateFilesystemContext(source, options.Filesystem, parsed); diagnostic != nil {
 				return input, []Diagnostic{*diagnostic}
@@ -48,7 +50,12 @@ func assembleBaseInputTracked[T any](source Source, options Options[T], accounti
 		if diagnostics := disabledDirectiveDiagnostics(input.document, Path{}); len(diagnostics) != 0 {
 			return input, diagnostics
 		}
-		if accounting.sources+1 > limits.sources || accounting.bytes+parsed.bytes > limits.bytes || accounting.nodes+countInputNodes(input.document) > limits.nodes {
+		_, placed, err := placeDocument(input.document, destination, input.document.Location, parseLimits.document.Depth, parseLimits.document.Nodes)
+		if err != nil {
+			return input, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, parsed.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
+		}
+		input.document = placed
+		if accounting.sources+1 > limits.sources || accounting.bytes+parsed.bytes > limits.bytes || accounting.nodes+countInputNodes(parsed.document) > limits.nodes {
 			return input, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, parsed.source.ID, "configuration exceeds the aggregate input limit")}
 		}
 		if diagnostic := registerAcquiredSource(source, options.Filesystem, parsed.source, visited, hostToken); diagnostic != nil {
@@ -56,7 +63,7 @@ func assembleBaseInputTracked[T any](source Source, options Options[T], accounti
 		}
 		accounting.sources++
 		accounting.bytes += parsed.bytes
-		accounting.nodes += countInputNodes(input.document)
+		accounting.nodes += countInputNodes(parsed.document)
 	}
 	if !activeOverlay(options.Overlay) || options.Imports {
 		contexts, diagnostic := contextsForAcquiredSources(source, options.Filesystem, parsed.source, input.sources)

internal/engine/load_request.go

@@ -17,9 +17,10 @@ type loadRequest[T any] struct {
 }
 
 type loadInput struct {
-	source     Source
+	source      Source
-	imports    bool
+	imports     bool
-	filesystem *FilesystemContext
+	filesystem  *FilesystemContext
+	destination Path
 }
 
 type loadGroup struct{ inputs []loadInput }
@@ -27,12 +28,23 @@ type loadGroup struct{ inputs []loadInput }
 func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
 	request.ordinary = false
 	var diagnostics []Diagnostic
+	if normalized, err := normalizeDestination(request.main.destination); err != nil {
+		diagnostics = append(diagnostics, invalidDestinationDiagnostic(request.main.destination))
+	} else {
+		request.main.destination = normalized
+	}
 	_, mainDiagnostics := selectOptions(request.main.source, []Options[T]{{
 		Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem,
 	}})
 	diagnostics = append(diagnostics, mainDiagnostics...)
-	for _, group := range request.underlays {
+	for groupIndex := range request.underlays {
-		for _, input := range group.inputs {
+		for inputIndex := range request.underlays[groupIndex].inputs {
+			input := &request.underlays[groupIndex].inputs[inputIndex]
+			if normalized, err := normalizeDestination(input.destination); err != nil {
+				diagnostics = append(diagnostics, invalidDestinationDiagnostic(input.destination))
+			} else {
+				input.destination = normalized
+			}
 			_, inputDiagnostics := selectOptions(input.source, []Options[T]{{Imports: input.imports, Filesystem: input.filesystem}})
 			diagnostics = append(diagnostics, inputDiagnostics...)
 		}
@@ -134,7 +146,7 @@ func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, account
 	var contexts map[string]ResourceContext
 	for _, contribution := range group.inputs {
 		options := Options[struct{}]{Imports: contribution.imports, Filesystem: contribution.filesystem, Definition: Definition[struct{}]{Version: rule}}
-		acquired, diagnostics := assembleBaseInputTracked(contribution.source, options, accounting, visited, hostToken)
+		acquired, diagnostics := assembleBaseInputTracked(contribution.source, options, contribution.destination, accounting, visited, hostToken)
 		for _, source := range acquired.sources {
 			source.Layer = LayerUnderlay
 			source.UnderlayIndex = index

internal/engine/records.go

@@ -186,6 +186,7 @@ type SourceInfo struct {
 	ID            string
 	Label         string
 	Path          string
+	Destination   Path
 	Layer         Layer
 	Kind          SourceKind
 	UnderlayIndex int
@@ -260,6 +261,7 @@ func cloneLocations(locations []Location) []Location {
 
 func cloneSourceInfo(source SourceInfo) SourceInfo {
 	source.ImportedFrom = cloneLocation(source.ImportedFrom)
+	source.Destination = source.Destination.clone()
 	return source
 }