Compose grouped underlays in shared load lifecycle

2026-09-29 17:52:20 +00:00
parent 90c6e8ed7e
commit 1cdf90d831
7 changed files with 494 additions and 25 deletions
--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -363,6 +363,16 @@ diagnostics remain stable. Run the everyday checker and engine race tests.

 ## Stage 6: Implement private grouped-underlay orchestration

+**Completed 2026-09-29.** Ordinary and grouped engine loads now use one private
+request lifecycle. Group-local strict composition and repetition tracking,
+load-wide source/byte/node/glob accounting, main-owned version checking,
+underlay layer/index metadata, and precedence-ordered assembly feed one final
+decode, resolution, validation, and publication. Engine tests cover two tiers,
+multiple contributions, source/history order, absent main/defaults-only loads,
+main/import and group conflicts, version protection, shared limits, reusable
+files across tiers, and replacement of invalid intermediate typed input. The
+everyday checker and engine race suite passed. No root layered API is exposed yet.
+
 1. Add private request/input/group structures matching the settled facade design.
   Adapt ordinary `Load` to the shared core while preserving its options and
   source-order behavior. Keep its existing validation order where observable.
--- a/internal/engine/imports.go
+++ b/internal/engine/imports.go
@@ -58,12 +58,16 @@ type importTraversal struct {
 }

 func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
+	return loadImportedDocumentsTracked(source, context, main, rule, limits, &importAccounting{}, nil, nil)
+}
+
+func loadImportedDocumentsTracked(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (*composedDocument, []SourceInfo, []Diagnostic) {
 	if nativeContext(source, context) {
 		access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
 		if diagnostic != nil {
 			return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
 		}
-		return loadImportedWithAccess(access, basePath, mainPath, main, rule, limits, &importAccounting{})
+		return loadImportedWithAccessTracked(access, basePath, mainPath, main, rule, limits, accounting, visited)
 	}
 	rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
 	if diagnostic != nil {
@@ -75,10 +79,17 @@ func loadImportedDocuments(source Source, context *FilesystemContext, main parse
 		diagnostic.Primary = nil
 		return nil, sourceInfos(main.source), []Diagnostic{diagnostic}
 	}
-	return loadImportedWithAccess(access, basePath, mainPath, main, rule, limits, &importAccounting{})
+	if hostToken != nil {
+		access.token = hostToken
+	}
+	return loadImportedWithAccessTracked(access, basePath, mainPath, main, rule, limits, accounting, visited)
 }

 func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
+	return loadImportedWithAccessTracked(access, basePath, mainPath, main, rule, limits, accounting, nil)
+}
+
+func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute) (*composedDocument, []SourceInfo, []Diagnostic) {
 	if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
 		_ = access.close()
 		return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
@@ -94,6 +105,14 @@ func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string,
 	if mainPath != "" {
 		mainInfo.Path = mainPath
 	}
+	mainKey := access.identity(mainPath)
+	if mainPath == "" {
+		mainKey = access.syntheticIdentity()
+	}
+	if previous, repeated := visited[mainKey]; repeated {
+		_ = access.close()
+		return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})}
+	}
 	accounting.sources++
 	accounting.bytes += main.bytes
 	accounting.nodes += countInputNodes(main.document)
@@ -101,7 +120,10 @@ func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string,
 		access: access, accounting: accounting, limits: limits, rule: rule,
 		documents: make([]*document.Node, 0, 1),
 		sources:   []SourceInfo{mainInfo},
-		active:    make(map[resourceIdentity]int), visited: make(map[resourceIdentity]importRoute),
+		active:    make(map[resourceIdentity]int), visited: visited,
+	}
+	if state.visited == nil {
+		state.visited = make(map[resourceIdentity]importRoute)
 	}
 	if diagnostic := state.checkInitialLimits(); diagnostic != nil {
 		_ = access.close()
@@ -109,10 +131,8 @@ func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string,
 	}

 	mainIdentity := mainPath
-	mainKey := access.identity(mainPath)
 	if mainIdentity == "" {
 		mainIdentity = "<selected-source>"
-		mainKey = access.syntheticIdentity()
 	}
 	mainRoute := importRoute{canonical: mainIdentity}
 	state.active[mainKey] = 0
--- a/internal/engine/load.go
+++ b/internal/engine/load.go
@@ -121,14 +121,27 @@ func Load[T any](source Source, secrets SecretBinding, supplied ...Options[T]) (
 }

 func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup secretLookup, supplied ...Options[T]) (*Loaded[T], error) {
-	accumulator := diagnosticAccumulator{}
 	options, diagnostics := selectOptions(source, supplied)
-	accumulator.add(diagnostics...)
 	if len(diagnostics) != 0 {
-		return nil, accumulator.loadError()
+		return nil, (&diagnosticAccumulator{diagnostics: diagnostics}).loadError()
+	}
+	request := loadRequest[T]{
+		main:       loadInput{source: source, imports: options.Imports, filesystem: options.Filesystem},
+		definition: options.Definition, overlay: options.Overlay,
+		overrides: options.Overrides, ordinary: true,
+	}
+	return executeLoadRequest(request, secrets, lookup)
+}
+
+func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
+	accumulator := diagnosticAccumulator{}
+	accounting := &importAccounting{}
+	mainOptions := Options[T]{Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem}
+	if request.ordinary {
+		mainOptions.Overlay = request.overlay
 	}

-	input, diagnostics := assembleBaseInput(source, options)
+	input, diagnostics := assembleBaseInputTracked(request.main.source, mainOptions, accounting, nil, nil)
 	accumulator.sources = input.sources
 	accumulator.add(diagnostics...)
 	if len(diagnostics) != 0 {
@@ -139,24 +152,28 @@ func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup se
 		accumulator.add(diagnosticFromSchemaIssue(schemaIssue))
 		return nil, accumulator.loadError()
 	}
-	versionDiagnostics := checkVersion(options.Definition.Version, input.document)
+	versionDiagnostics := checkVersion(request.definition.Version, input.document)
 	accumulator.add(versionDiagnostics...)
 	if hasErrors(versionDiagnostics) {
 		return nil, accumulator.loadError()
 	}
-	input, diagnostics = applyInputLayers(source, options, input)
+	if request.ordinary {
+		input, diagnostics = applyInputLayers(request.main.source, Options[T]{Definition: request.definition, Overlay: request.overlay, Filesystem: request.main.filesystem, Overrides: request.overrides}, input)
+	} else {
+		input, diagnostics = assembleLayeredInput(request, input, accounting)
+	}
 	accumulator.sources = input.sources
 	accumulator.add(diagnostics...)
 	if len(diagnostics) != 0 {
 		return nil, accumulator.loadError()
 	}

-	decoded, diagnostics := decodeLoadInput(schema, options.Definition, input)
+	decoded, diagnostics := decodeLoadInput(schema, request.definition, input)
 	accumulator.add(diagnostics...)
 	if len(diagnostics) != 0 {
 		return nil, accumulator.loadError()
 	}
-	resolved, diagnostics := resolveLoadInput(schema, options.Definition, input, decoded)
+	resolved, diagnostics := resolveLoadInput(schema, request.definition, input, decoded)
 	accumulator.add(diagnostics...)
 	if hasErrors(diagnostics) {
 		return nil, accumulator.loadError()
@@ -177,8 +194,8 @@ func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup se
 	if hasErrors(secretResolution.diagnostics) {
 		return nil, accumulator.loadError()
 	}
-	if options.Definition.Validate != nil {
-		diagnostics = runValidation(&configuration, options.Definition.Validate, schema, resolved.final, input.document, secretResolution.lookups)
+	if request.definition.Validate != nil {
+		diagnostics = runValidation(&configuration, request.definition.Validate, schema, resolved.final, input.document, secretResolution.lookups)
 		accumulator.add(diagnostics...)
 		if hasErrors(diagnostics) {
 			return nil, accumulator.loadError()
@@ -187,5 +204,5 @@ func loadWithSecretLookup[T any](source Source, secrets SecretBinding, lookup se
 	resolved.configuration = configuration
 	// Publication detaches equal-valued validator replacements, preserving the
 	// validated secret state, then finalizes metadata and sensitivity.
-	return publishLoadResult(schema, options.Definition, resolved, secretResolution.references, &accumulator)
+	return publishLoadResult(schema, request.definition, resolved, secretResolution.references, &accumulator)
 }
--- a/internal/engine/load_input.go
+++ b/internal/engine/load_input.go
@@ -1,6 +1,10 @@
 package engine

-import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+import (
+	"strconv"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)

 // assembledInput is the source/document handoff across schema and version checks.
 type assembledInput struct {
@@ -11,14 +15,21 @@ type assembledInput struct {
 }

 func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput, []Diagnostic) {
-	parsed, diagnostics := parseSource(source, "source-1", defaultParseLimits())
+	return assembleBaseInputTracked(source, options, &importAccounting{}, nil, nil)
+}
+
+func assembleBaseInputTracked[T any](source Source, options Options[T], accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (assembledInput, []Diagnostic) {
+	limits := defaultImportLimits()
+	parseLimits := defaultParseLimits()
+	parseLimits.source.remaining = limits.bytes - accounting.bytes
+	parsed, diagnostics := parseSource(source, "source-"+strconv.Itoa(accounting.sources+1), parseLimits)
 	input := assembledInput{parsed: parsed, document: parsed.document}
 	if len(diagnostics) != 0 {
 		input.sources = sourceInfos(parsed.source)
 		return input, diagnostics
 	}
 	if options.Imports {
-		input.composition, input.sources, diagnostics = loadImportedDocuments(source, options.Filesystem, parsed, options.Definition.Version, defaultImportLimits())
+		input.composition, input.sources, diagnostics = loadImportedDocumentsTracked(source, options.Filesystem, parsed, options.Definition.Version, limits, accounting, visited, hostToken)
 		if len(diagnostics) != 0 {
 			return input, diagnostics
 		}
@@ -35,10 +46,37 @@ func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput
 		if diagnostics := disabledDirectiveDiagnostics(input.document, Path{}); len(diagnostics) != 0 {
 			return input, diagnostics
 		}
+		if accounting.sources+1 > limits.sources || accounting.bytes+parsed.bytes > limits.bytes || accounting.nodes+countInputNodes(input.document) > limits.nodes {
+			return input, []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, parsed.source.ID, "configuration exceeds the aggregate input limit")}
+		}
+		if diagnostic := registerAcquiredSource(source, options.Filesystem, parsed.source, visited, hostToken); diagnostic != nil {
+			return input, []Diagnostic{*diagnostic}
+		}
+		accounting.sources++
+		accounting.bytes += parsed.bytes
+		accounting.nodes += countInputNodes(input.document)
 	}
 	return input, nil
 }

+func registerAcquiredSource(source Source, context *FilesystemContext, info *SourceInfo, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) *Diagnostic {
+	if visited == nil || info == nil || info.Path == "" {
+		return nil
+	}
+	identity := resourceIdentity{namespace: hostToken, path: info.Path}
+	if source.kind == sourceFilesystem {
+		identity.namespace = source.namespace.token
+	} else if context != nil && context.FS != nil {
+		identity.namespace = context.FS.namespace.token
+	}
+	if previous, repeated := visited[identity]; repeated {
+		value := importRepeatedDiagnostic(Location{SourceID: info.ID}, previous, importRoute{})
+		return &value
+	}
+	visited[identity] = importRoute{canonical: info.Path}
+	return nil
+}
+
 func applyInputLayers[T any](source Source, options Options[T], input assembledInput) (assembledInput, []Diagnostic) {
 	if !activeOverlay(options.Overlay) && len(options.Overrides) == 0 {
 		return input, nil
--- a/internal/engine/load_request.go
+++ b/internal/engine/load_request.go
@@ -0,0 +1,171 @@
+package engine
+
+import (
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
+
+// loadRequest keeps source acquisition separate from precedence assembly.
+// The ordinary entry point and grouped entry point share the final lifecycle.
+type loadRequest[T any] struct {
+	main              loadInput
+	underlays         []loadGroup
+	definition        Definition[T]
+	overlay           string
+	overlayFilesystem *FilesystemContext
+	overrides         []Override
+	ordinary          bool
+}
+
+type loadInput struct {
+	source     Source
+	imports    bool
+	filesystem *FilesystemContext
+}
+
+type loadGroup struct{ inputs []loadInput }
+
+func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBinding, lookup secretLookup) (*Loaded[T], error) {
+	request.ordinary = false
+	var diagnostics []Diagnostic
+	_, mainDiagnostics := selectOptions(request.main.source, []Options[T]{{
+		Definition: request.definition, Imports: request.main.imports, Filesystem: request.main.filesystem,
+	}})
+	diagnostics = append(diagnostics, mainDiagnostics...)
+	for _, group := range request.underlays {
+		for _, input := range group.inputs {
+			_, inputDiagnostics := selectOptions(input.source, []Options[T]{{Imports: input.imports, Filesystem: input.filesystem}})
+			diagnostics = append(diagnostics, inputDiagnostics...)
+		}
+	}
+	if !validOverlayName(request.overlay) {
+		diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "overlay name must contain only ASCII letters, digits, underscore, or hyphen"))
+	}
+	if activeOverlay(request.overlay) {
+		if request.overlayFilesystem == nil {
+			diagnostics = append(diagnostics, optionDiagnostic(CodeOptionsInvalid, "selected overlay requires explicit filesystem context"))
+		} else {
+			diagnostics = append(diagnostics, filesystemOptionDiagnostics(Source{kind: sourceBytes}, request.overlayFilesystem)...)
+		}
+	}
+	if len(diagnostics) != 0 {
+		return nil, (&diagnosticAccumulator{diagnostics: canonicalDiagnostics(diagnostics)}).loadError()
+	}
+	return executeLoadRequest(request, secrets, lookup)
+}
+
+func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
+	groups := make([]*composedDocument, 0, len(request.underlays))
+	for index, group := range request.underlays {
+		composition, sources, diagnostics := acquireUnderlayGroup(group, index+1, request.definition.Version, accounting)
+		input.sources = append(input.sources, sources...)
+		if len(diagnostics) != 0 {
+			return input, diagnostics
+		}
+		if composition != nil {
+			groups = append(groups, composition)
+		}
+	}
+
+	var assembly *assemblyAccumulator
+	for _, group := range groups {
+		if assembly == nil {
+			assembly = newAssemblyAccumulator(group)
+			continue
+		}
+		if diagnostics := assembly.applyOverlay(group.root, group.origins, group.applications, LayerUnderlay); len(diagnostics) != 0 {
+			return input, diagnostics
+		}
+	}
+	if input.document != nil {
+		mainComposition, diagnostics := compositionForInput(input)
+		if len(diagnostics) != 0 {
+			return input, diagnostics
+		}
+		if assembly == nil {
+			assembly = newAssemblyAccumulator(mainComposition)
+		} else if diagnostics := assembly.applyOverlay(mainComposition.root, mainComposition.origins, mainComposition.applications, LayerBase); len(diagnostics) != 0 {
+			return input, diagnostics
+		}
+	}
+	if assembly == nil && (activeOverlay(request.overlay) || len(request.overrides) != 0) {
+		composition, _ := compositionForInput(assembledInput{})
+		assembly = newAssemblyAccumulator(composition)
+	}
+	if assembly == nil {
+		return input, nil
+	}
+	assembly.current.inputBytes = accounting.bytes
+	assembly.current.inputNodes = accounting.nodes
+	if activeOverlay(request.overlay) {
+		// The selected overlay context is independent of the main input. This
+		// synthetic descriptor creates no source record or input contribution.
+		selection := Source{kind: sourceBytes}
+		selected := parsedSource{document: &document.Node{Kind: document.Mapping}, source: &SourceInfo{}}
+		var diagnostics []Diagnostic
+		input.sources, diagnostics = loadSelectedOverlayInto(selection, request.overlayFilesystem, selected, assembly, input.sources, request.overlay, request.definition.Version, defaultImportLimits())
+		if len(diagnostics) != 0 {
+			return input, diagnostics
+		}
+	}
+	if len(request.overrides) != 0 {
+		var diagnostics []Diagnostic
+		input.sources, diagnostics = assembly.applyOverrides(input.sources, request.overrides, request.definition.Version, defaultImportLimits())
+		if len(diagnostics) != 0 {
+			return input, diagnostics
+		}
+	}
+	input.composition = assembly.current
+	input.document = assembly.current.root
+	return input, nil
+}
+
+func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
+	visited := make(map[resourceIdentity]importRoute)
+	hostToken := &namespaceToken{}
+	var documents []*document.Node
+	var sources []SourceInfo
+	for _, contribution := range group.inputs {
+		options := Options[struct{}]{Imports: contribution.imports, Filesystem: contribution.filesystem, Definition: Definition[struct{}]{Version: rule}}
+		acquired, diagnostics := assembleBaseInputTracked(contribution.source, options, accounting, visited, hostToken)
+		for _, source := range acquired.sources {
+			source.Layer = LayerUnderlay
+			source.UnderlayIndex = index
+			sources = append(sources, source)
+		}
+		if len(diagnostics) != 0 {
+			return nil, sources, diagnostics
+		}
+		if acquired.document == nil {
+			continue
+		}
+		if node := fragmentVersionClaim(acquired.document, rule); node != nil {
+			location := publicLocation(node.Location)
+			return nil, sources, []Diagnostic{{
+				Code: CodeUnderlayVersion, Severity: SeverityError, Path: rule.Path.clone(),
+				Message: "underlay inputs cannot contribute to the schema-version path", Primary: &location,
+			}}
+		}
+		if acquired.composition != nil {
+			for _, application := range acquired.composition.applications {
+				documents = append(documents, application.value)
+			}
+		} else {
+			documents = append(documents, acquired.document)
+		}
+	}
+	if len(documents) == 0 {
+		return nil, sources, nil
+	}
+	composition, diagnostics := composeDocuments(documents...)
+	if len(diagnostics) != 0 {
+		return nil, sources, diagnostics
+	}
+	for pointer, origin := range composition.origins {
+		origin.Layer = LayerUnderlay
+		composition.origins[pointer] = origin
+	}
+	for index := range composition.applications {
+		composition.applications[index].layer = LayerUnderlay
+	}
+	return composition, sources, nil
+}
--- a/internal/engine/load_request_test.go
+++ b/internal/engine/load_request_test.go
@@ -0,0 +1,210 @@
+package engine
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+type groupedConfig struct {
+	Version  string            `yaml:"version"`
+	Settings map[string]string `yaml:"settings"`
+	Port     int               `yaml:"port"`
+	Derived  string            `yaml:"derived"`
+}
+
+func testGroupedLoad[T any](request loadRequest[T]) (*testResult[T], error) {
+	loaded, err := loadGroupedWithSecretLookup(request, testSecretBinding(), func(string) (string, bool) { return "", false })
+	return testLoaded(loaded), err
+}
+
+func TestGroupedLoadPrecedenceSourceOrderAndSingleLifecycle(t *testing.T) {
+	filesystem := NewFilesystem("bundle", fstest.MapFS{
+		"main.yml":          &fstest.MapFile{Data: []byte("configkit: {imports: [{path: imported.yml}]}\nversion: v1\nport: 5\nsettings: {shared: main}\n")},
+		"imported.yml":      &fstest.MapFile{Data: []byte("settings: {imported: included}\n")},
+		"overlays/test.yml": &fstest.MapFile{Data: []byte("settings: {shared: overlay}\n")},
+	})
+	checks := struct{ version, defaults, resolution, validation int }{}
+	request := loadRequest[groupedConfig]{
+		main: loadInput{source: filesystem.File("main.yml"), imports: true},
+		underlays: []loadGroup{
+			{inputs: []loadInput{
+				{source: Mapping("low-a", map[string]any{"settings": map[string]any{"shared": "low", "a": "one"}, "port": "invalid until replaced"})},
+				{source: Bytes("low-b", []byte("settings: {b: two}\n"))},
+			}},
+			{inputs: []loadInput{{source: Mapping("middle", map[string]any{"settings": map[string]any{"shared": "middle", "c": "three"}})}}},
+		},
+		definition: Definition[groupedConfig]{
+			Version: &VersionRule{Path: Keys("version"), Check: func(version Version, validation *Validation) error {
+				checks.version++
+				if version.Text != "v1" {
+					t.Errorf("version = %#v", version)
+				}
+				return nil
+			}},
+			Defaults: func() groupedConfig { checks.defaults++; return groupedConfig{Derived: "initial"} },
+			Resolve: func(config *groupedConfig, resolution *Resolution) error {
+				checks.resolution++
+				Default(resolution, Keys("derived"), &config.Derived, "resolved", "set derived")
+				return nil
+			},
+			Validate: func(config *groupedConfig, validation *Validation) { checks.validation++ },
+		},
+		overlay: "test", overlayFilesystem: &FilesystemContext{Root: ".", BaseDir: ".", FS: &filesystem},
+		overrides: []Override{{Path: Keys("settings", "shared"), Value: "cli", Flag: "--shared"}},
+	}
+	result, err := testGroupedLoad(request)
+	if err != nil {
+		t.Fatalf("load: %v, diagnostics %#v", err, err.(*LoadError).Diagnostics())
+	}
+	wantSettings := map[string]string{"shared": "cli", "a": "one", "b": "two", "c": "three", "imported": "included"}
+	if result.Config.Port != 5 || !reflect.DeepEqual(result.Config.Settings, wantSettings) {
+		t.Fatalf("config = %#v", result.Config)
+	}
+	if checks != (struct{ version, defaults, resolution, validation int }{1, 1, 1, 1}) {
+		t.Fatalf("lifecycle calls = %#v", checks)
+	}
+	sources := result.Sources()
+	if len(sources) != 7 {
+		t.Fatalf("sources = %#v", sources)
+	}
+	wantLayers := []Layer{LayerBase, LayerBase, LayerUnderlay, LayerUnderlay, LayerUnderlay, LayerOverlay, LayerCLI}
+	wantIndexes := []int{0, 0, 1, 1, 2, 0, 0}
+	for index, source := range sources {
+		if source.ID != "source-"+string(rune('1'+index)) || source.Layer != wantLayers[index] || source.UnderlayIndex != wantIndexes[index] {
+			t.Errorf("source %d = %#v", index, source)
+		}
+	}
+	info, ok := result.Metadata().Lookup(Keys("settings", "shared"))
+	if !ok || info.Origin.Layer != LayerCLI {
+		t.Fatalf("effective shared metadata = %#v, %v", info, ok)
+	}
+	explanation, ok := result.Explain(Keys("settings", "shared"))
+	if !ok || len(explanation.History) < 5 {
+		t.Fatalf("shared explanation = %#v, %v", explanation, ok)
+	}
+	var layers []Layer
+	for _, change := range explanation.History {
+		if change.Origin.Layer != LayerDefault && (len(layers) == 0 || layers[len(layers)-1] != change.Origin.Layer) {
+			layers = append(layers, change.Origin.Layer)
+		}
+	}
+	if !reflect.DeepEqual(layers, []Layer{LayerUnderlay, LayerBase, LayerOverlay, LayerCLI}) {
+		t.Fatalf("history layers = %#v", layers)
+	}
+}
+
+func TestGroupedLoadAbsentMainAndVersionProtection(t *testing.T) {
+	var observed Version
+	request := loadRequest[groupedConfig]{
+		main:      loadInput{source: NoSource()},
+		underlays: []loadGroup{{}, {inputs: []loadInput{{source: Bytes("underlay", []byte("port: 7\n"))}}}},
+		definition: Definition[groupedConfig]{Version: &VersionRule{Path: Keys("version"), Check: func(version Version, validation *Validation) error {
+			observed = version
+			return nil
+		}}},
+	}
+	result, err := testGroupedLoad(request)
+	if err != nil || result.Config.Port != 7 || observed.Present || len(result.Sources()) != 1 || result.Sources()[0].UnderlayIndex != 2 {
+		t.Fatalf("underlay-only load = %#v, version %#v, %v", result, observed, err)
+	}
+	request.underlays[1].inputs[0].source = Bytes("underlay", []byte("version: forbidden\n"))
+	_, err = testGroupedLoad(request)
+	if got := groupedErrorCode(t, err); got != CodeUnderlayVersion {
+		t.Fatalf("underlay version code = %s", got)
+	}
+	filesystem := NewFilesystem("bundle", fstest.MapFS{
+		"main.yml":     &fstest.MapFile{Data: []byte("configkit: {imports: [{path: fragment.yml}]}\n")},
+		"fragment.yml": &fstest.MapFile{Data: []byte("version: forbidden\n")},
+	})
+	request.underlays[1].inputs[0] = loadInput{source: filesystem.File("main.yml"), imports: true}
+	_, err = testGroupedLoad(request)
+	if got := groupedErrorCode(t, err); got != CodeImportVersion {
+		t.Fatalf("import version code = %s", got)
+	}
+	request.underlays = []loadGroup{{}}
+	request.definition.Defaults = func() groupedConfig { return groupedConfig{Port: 11} }
+	result, err = testGroupedLoad(request)
+	if err != nil || result.Config.Port != 11 || len(result.Sources()) != 0 {
+		t.Fatalf("defaults-only load = %#v, %v", result, err)
+	}
+}
+
+func TestGroupedLoadStrictConflictsRepetitionAndBudgets(t *testing.T) {
+	main := loadInput{source: Bytes("main", []byte("port: 5\n"))}
+	request := loadRequest[groupedConfig]{
+		main: main,
+		underlays: []loadGroup{{inputs: []loadInput{
+			{source: Bytes("left", []byte("port: 1\n"))},
+			{source: Bytes("right", []byte("port: 2\n"))},
+		}}},
+	}
+	_, err := testGroupedLoad(request)
+	if got := groupedErrorCode(t, err); got != CodeComposeConflict {
+		t.Fatalf("strict conflict code = %s", got)
+	}
+	conflicting := NewFilesystem("bundle", fstest.MapFS{
+		"main.yml":     &fstest.MapFile{Data: []byte("configkit: {imports: [{path: fragment.yml}]}\nport: 1\n")},
+		"fragment.yml": &fstest.MapFile{Data: []byte("port: 2\n")},
+	})
+	request.main = loadInput{source: conflicting.File("main.yml"), imports: true}
+	request.underlays = []loadGroup{{inputs: []loadInput{{source: Bytes("lower", []byte("port: 3\n"))}}}}
+	_, err = testGroupedLoad(request)
+	if got := groupedErrorCode(t, err); got != CodeComposeConflict {
+		t.Fatalf("main/import conflict code = %s", got)
+	}
+	request.main = main
+	filesystem := NewFilesystem("bundle", fstest.MapFS{"same.yml": &fstest.MapFile{Data: []byte("port: 1\n")}})
+	file := filesystem.File("same.yml")
+	request.underlays = []loadGroup{{inputs: []loadInput{{source: file}, {source: file}}}}
+	_, err = testGroupedLoad(request)
+	if got := groupedErrorCode(t, err); got != CodeImportRepeated {
+		t.Fatalf("same-group repetition code = %s", got)
+	}
+	request.underlays = []loadGroup{{inputs: []loadInput{{source: file}}}, {inputs: []loadInput{{source: file}}}}
+	if result, err := testGroupedLoad(request); err != nil || result.Config.Port != 5 || len(result.Sources()) != 3 {
+		t.Fatalf("cross-group reuse = %#v, %v", result, err)
+	}
+
+	accounting := &importAccounting{sources: maxImportSources - 1}
+	group := loadGroup{inputs: []loadInput{{source: Mapping("first", map[string]any{})}, {source: Mapping("second", map[string]any{})}}}
+	_, _, diagnostics := acquireUnderlayGroup(group, 1, nil, accounting)
+	if len(diagnostics) == 0 || diagnostics[0].Code != CodeSourceLimitExceeded {
+		t.Fatalf("shared source budget = %#v", diagnostics)
+	}
+	accounting = &importAccounting{bytes: maxTotalBytes - 2}
+	group.inputs = []loadInput{{source: Mapping("first", map[string]any{"port": "one"})}}
+	_, _, diagnostics = acquireUnderlayGroup(group, 1, nil, accounting)
+	if len(diagnostics) == 0 || diagnostics[0].Code != CodeSourceLimitExceeded {
+		t.Fatalf("shared byte budget = %#v", diagnostics)
+	}
+}
+
+func TestGroupedLoadInvalidIntermediateTypeCanBeReplaced(t *testing.T) {
+	request := loadRequest[groupedConfig]{
+		main:      loadInput{source: Bytes("main", []byte("port: 9\n"))},
+		underlays: []loadGroup{{inputs: []loadInput{{source: Bytes("low", []byte("port: wrong\n"))}}}},
+	}
+	result, err := testGroupedLoad(request)
+	if err != nil || result.Config.Port != 9 {
+		t.Fatalf("final typed decode = %#v, %v", result, err)
+	}
+	request.main.source = Bytes("main", []byte("port: wrong\n"))
+	_, err = testGroupedLoad(request)
+	if got := groupedErrorCode(t, err); got != CodeDecodeType && got != CodeDecodeRange {
+		t.Fatalf("final invalid type code = %s", got)
+	}
+}
+
+func groupedErrorCode(t *testing.T, err error) Code {
+	t.Helper()
+	loadError, ok := err.(*LoadError)
+	if !ok || len(loadError.Diagnostics()) == 0 {
+		t.Fatalf("error = %#v", err)
+	}
+	if strings.Contains(loadError.Diagnostics()[0].Message, "wrong") {
+		t.Fatal("raw value escaped diagnostics")
+	}
+	return loadError.Diagnostics()[0].Code
+}
--- a/internal/engine/records.go
+++ b/internal/engine/records.go
@@ -6,6 +6,7 @@ type Layer string
 const (
 	LayerDefault    Layer = "default"
 	LayerBase       Layer = "base"
+	LayerUnderlay   Layer = "underlay"
 	LayerOverlay    Layer = "overlay"
 	LayerCLI        Layer = "cli"
 	LayerResolution Layer = "resolution"
@@ -101,6 +102,7 @@ const (
 	CodeImportRepeated      Code = "import.repeated"
 	CodeImportLimitExceeded Code = "import.limit_exceeded"
 	CodeImportVersion       Code = "import.version"
+	CodeUnderlayVersion     Code = "underlay.version"

 	CodeOverlayInvalid       Code = "overlay.invalid"
 	CodeOverlayMissing       Code = "overlay.missing"
@@ -173,12 +175,13 @@ type Location struct {
 // namespace-relative filesystem path; synthetic sources have no Path.
 // ImportedFrom identifies the declaration that included this source.
 type SourceInfo struct {
-	ID           string
-	Label        string
-	Path         string
-	Layer        Layer
-	Kind         SourceKind
-	ImportedFrom *Location
+	ID            string
+	Label         string
+	Path          string
+	Layer         Layer
+	Kind          SourceKind
+	UnderlayIndex int
+	ImportedFrom  *Location
 }

 // Diagnostic is a safe structured problem report. Message and source labels must