Check source versions in direct underlays

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Implement shared source version checks
 
-Status: Stages 1–5 complete; direct underlay integration has not started.
+Status: Stages 1–6 complete; selected overlay integration has not started.
 
 Audience: a Luna coding agent implementing one numbered stage per prompt, in
 order. The [feature roadmap](source-version-checks.md) owns target behavior and
@@ -379,6 +379,26 @@ No environmental limitation affected validation; checks ran on Linux/amd64.
 Validate: grouped/underlay/destination/version tests and everyday check. Done:
 direct and imported underlays follow the same checking policy.
 
+Completion evidence: threaded the per-load version state through direct underlay
+group acquisition and invoked the shared processor with contribution semantics
+for each acquired underlay root. Its own processed payload/import declarations
+now drive placement and recursive imports, while remaining placed payload is
+still checked against the effective main-owned path before strict composition.
+Warnings are accumulated through underlay groups; source records retain their
+layer and group index, and the existing accounting, context, and reusable-source
+paths remain in place. Added coverage for distinct declarations within one
+strict group, warning retention across groups, rejected declarations beneath a
+later main value, required omissions, empty groups and NoSource, mounted nested
+headers/imports, and protected remaining payload. Existing default underlay
+version-protection coverage remains passing. These checks passed with Go
+`go1.26.8 linux/amd64`:
+
+- `GOTOOLCHAIN=local GOPROXY=off go test ./internal/engine -run 'SourceVersion|Version|Import|Underlay|Overlay|Destination' -count=1`
+- `GOTOOLCHAIN=local ./scripts/check.sh`
+- `git diff --check`
+
+No environmental limitation affected validation; checks ran on Linux/amd64.
+
 ## Stage 7 — Integrate selected overlays and final ownership checks
 
 1. Process overlay headers after parse/structural validation, before remaining

internal/engine/imports.go

@@ -67,12 +67,20 @@ func loadImportedDocumentsTracked(source Source, context *FilesystemContext, mai
 }
 
 func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
+	var processing *sourceVersionProcessingResult
+	if versionState != nil && versionState.mode != sourceVersionModeDefault {
+		processing = versionState.mainProcessing
+	}
+	return loadImportedDocumentsTrackedWithProcessing(source, context, main, destination, rule, limits, accounting, visited, hostToken, versionState, processing)
+}
+
+func loadImportedDocumentsTrackedWithProcessing(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) {
 	if nativeContext(source, context) {
 		access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
 		if diagnostic != nil {
 			return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
 		}
-		return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState)
+		return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
 	}
 	rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
 	if diagnostic != nil {
@@ -87,7 +95,7 @@ func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemC
 	if hostToken != nil {
 		access.token = hostToken
 	}
-	return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState)
+	return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
 }
 
 func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
@@ -99,6 +107,14 @@ func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath
 }
 
 func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
+	var processing *sourceVersionProcessingResult
+	if versionState != nil && versionState.mode != sourceVersionModeDefault {
+		processing = versionState.mainProcessing
+	}
+	return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
+}
+
+func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) {
 	if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
 		_ = access.close()
 		return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
@@ -151,9 +167,9 @@ func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath
 	var body *document.Node
 	var declarations []importDeclaration
 	var diagnostics []Diagnostic
-	if versionState != nil && versionState.mode != sourceVersionModeDefault && versionState.mainProcessing != nil {
+	if processing != nil {
-		body = versionState.mainProcessing.payload
+		body = processing.payload
-		declarations = versionState.mainProcessing.imports
+		declarations = processing.imports
 	} else {
 		body, declarations, diagnostics = splitImportDirectives(main.document)
 	}

internal/engine/load.go

@@ -176,6 +176,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
 	input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, versionState)
 	accumulator.sources = input.sources
 	accumulator.add(versionState.diagnostics...)
+	reportedVersionDiagnostics := len(versionState.diagnostics)
 	accumulator.add(diagnostics...)
 	if hasErrors(diagnostics) {
 		return nil, accumulator.loadError()
@@ -200,6 +201,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
 	} else {
 		input, diagnostics = assembleLayeredInput(request, input, accounting)
 	}
+	accumulator.add(versionState.diagnostics[reportedVersionDiagnostics:]...)
 	accumulator.sources = input.sources
 	accumulator.add(diagnostics...)
 	if len(diagnostics) != 0 {

internal/engine/load_input.go

@@ -26,6 +26,10 @@ func assembleBaseInputTracked[T any](source Source, options Options[T], destinat
 }
 
 func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (assembledInput, []Diagnostic) {
+	return assembleBaseInputTrackedForRole(source, options, destination, accounting, visited, hostToken, versionState, sourceVersionMain)
+}
+
+func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, role sourceVersionRole) (assembledInput, []Diagnostic) {
 	limits := defaultImportLimits()
 	parseLimits := defaultParseLimits()
 	parseLimits.source.remaining = limits.bytes - accounting.bytes
@@ -47,25 +51,32 @@ func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T
 		}
 		return input, diagnostics
 	}
+	var versionProcessing *sourceVersionProcessingResult
 	if versionState != nil && versionState.mode != sourceVersionModeDefault {
 		var sourceID string
 		if parsed.source != nil {
 			sourceID = parsed.source.ID
 		}
-		processed, failures := versionState.process(parsed.document, sourceVersionMain, destination, sourceID)
+		processed, failures := versionState.process(parsed.document, role, destination, sourceID)
 		if len(failures) != 0 {
 			return input, failures
 		}
-		versionState.mainProcessing = &processed
+		versionProcessing = &processed
+		if role == sourceVersionMain {
+			versionState.mainProcessing = &processed
+		}
 	}
 	if options.Imports {
-		input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithVersion(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState)
+		input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessing(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing)
 		if len(diagnostics) != 0 {
 			return input, diagnostics
 		}
 		input.document = input.composition.root
 	} else {
 		input.sources = sourceInfos(parsed.source)
+		if versionProcessing != nil {
+			input.document = versionProcessing.payload
+		}
 		if options.Filesystem != nil && !activeOverlay(options.Overlay) {
 			if diagnostic := validateFilesystemContext(source, options.Filesystem, parsed); diagnostic != nil {
 				return input, []Diagnostic{*diagnostic}

internal/engine/load_request.go

@@ -79,7 +79,7 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
 func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
 	groups := make([]*composedDocument, 0, len(request.underlays))
 	for index, group := range request.underlays {
-		composition, sources, contexts, diagnostics := acquireUnderlayGroup(group, index+1, request.protectionVersion(), accounting)
+		composition, sources, contexts, diagnostics := acquireUnderlayGroupWithVersion(group, index+1, request.protectionVersion(), accounting, input.versionState)
 		input.sources = append(input.sources, sources...)
 		mergeResourceContexts(&input.contexts, contexts)
 		if len(diagnostics) != 0 {
@@ -149,6 +149,10 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
 }
 
 func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
+	return acquireUnderlayGroupWithVersion(group, index, rule, accounting, nil)
+}
+
+func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
 	initialBytes, initialNodes := accounting.bytes, accounting.nodes
 	visited := make(map[resourceIdentity]importRoute)
 	hostToken := &namespaceToken{}
@@ -157,7 +161,7 @@ func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, account
 	var contexts map[string]ResourceContext
 	for _, contribution := range group.inputs {
 		options := Options[struct{}]{Imports: contribution.imports, Filesystem: contribution.filesystem, Definition: Definition[struct{}]{Version: rule}}
-		acquired, diagnostics := assembleBaseInputTracked(contribution.source, options, contribution.destination, accounting, visited, hostToken)
+		acquired, diagnostics := assembleBaseInputTrackedForRole(contribution.source, options, contribution.destination, accounting, visited, hostToken, versionState, sourceVersionContribution)
 		for _, source := range acquired.sources {
 			source.Layer = LayerUnderlay
 			source.UnderlayIndex = index

internal/engine/source_version_test.go

@@ -2,6 +2,7 @@ package engine
 
 import (
 	"errors"
+	"path/filepath"
 	"testing"
 
 	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
@@ -754,6 +755,225 @@ configkit:
 	})
 }
 
+func TestSourceVersionUnderlaysCheckDistinctMountedDeclarations(t *testing.T) {
+	t.Parallel()
+
+	var checked []string
+	definition := Definition[map[string]any]{Version: &VersionRule{
+		Path: Keys("meta", "version"),
+		Check: func(version Version, validation *Validation) error {
+			checked = append(checked, version.Text)
+			if version.Text == "peer-a" || version.Text == "peer-c" {
+				validation.Warning(Path{}, "accepted with advisory")
+			}
+			return nil
+		},
+	}}
+	main := Input{
+		Source:      Bytes("main", []byte("meta: {version: main-v}\nservice: primary\n")),
+		Destination: Keys("service"),
+	}
+	first := Input{
+		Source:      Bytes("peer-a", []byte("meta:\n  version: peer-a\nvalues: {first: one}\n")),
+		Destination: Keys("defaults", "first"),
+	}
+	second := Input{
+		Source:      Bytes("peer-b", []byte("meta:\n  version: peer-b\nvalues: {second: two}\n")),
+		Destination: Keys("defaults", "second"),
+	}
+	third := Input{
+		Source:      Bytes("peer-c", []byte("meta:\n  version: peer-c\nvalues: {third: three}\n")),
+		Destination: Keys("defaults", "third"),
+	}
+	loaded, err := LoadLayered(main, testSecretBinding(), LayeredOptions[map[string]any]{
+		Definition: definition, SourceVersionMode: sourceVersionModeRequired,
+		Underlays: []Underlay{{Inputs: []Input{first, second}}, {Inputs: []Input{third}}},
+	})
+	if err != nil {
+		t.Fatalf("LoadLayered: %v", err)
+	}
+	if len(checked) != 4 {
+		t.Fatalf("checked declarations = %#v, want main and all underlays", checked)
+	}
+	for _, want := range []string{"main-v", "peer-a", "peer-b", "peer-c"} {
+		found := false
+		for _, got := range checked {
+			found = found || got == want
+		}
+		if !found {
+			t.Errorf("checked declarations %#v do not include %q", checked, want)
+		}
+	}
+	if len(loaded.Inspection.Diagnostics()) != 2 || loaded.Inspection.Diagnostics()[0].Code != CodeVersionWarning || loaded.Inspection.Diagnostics()[1].Code != CodeVersionWarning {
+		t.Fatalf("accepted underlay warning was not retained: %#v", loaded.Inspection.Diagnostics())
+	}
+	sources := loaded.Inspection.Sources()
+	if len(sources) != 4 || sources[1].Layer != LayerUnderlay || sources[1].UnderlayIndex != 1 || sources[2].Layer != LayerUnderlay || sources[2].UnderlayIndex != 1 || sources[3].Layer != LayerUnderlay || sources[3].UnderlayIndex != 2 {
+		t.Fatalf("underlay source records = %#v", sources)
+	}
+	defaults, ok := loaded.Config["defaults"].(map[string]any)
+	if !ok {
+		t.Fatalf("defaults = %#v", loaded.Config["defaults"])
+	}
+	firstValue, ok := defaults["first"].(map[string]any)
+	if !ok || firstValue["values"] == nil {
+		t.Fatalf("first mounted contribution = %#v", defaults["first"])
+	}
+	if meta, ok := firstValue["meta"].(map[string]any); !ok || len(meta) != 0 {
+		t.Fatalf("consumed nested declaration did not preserve its empty ancestor: %#v", firstValue["meta"])
+	}
+}
+
+func TestSourceVersionUnderlaysRejectDeclarationsHiddenByMain(t *testing.T) {
+	t.Parallel()
+
+	var checked []string
+	loaded, err := LoadLayered(Input{Source: Bytes("main", []byte("meta: {version: main-v}\nservice: primary\n"))}, testSecretBinding(), LayeredOptions[map[string]any]{
+		Definition: Definition[map[string]any]{Version: &VersionRule{
+			Path: Keys("meta", "version"),
+			Check: func(version Version, _ *Validation) error {
+				checked = append(checked, version.Text)
+				if version.Text == "rejected-peer" {
+					return errors.New("peer declaration is not accepted")
+				}
+				return nil
+			},
+		}},
+		SourceVersionMode: sourceVersionModeIfPresent,
+		Underlays:         []Underlay{{Inputs: []Input{{Source: Bytes("underlay", []byte("meta: {version: rejected-peer}\nservice: shadowed\n"))}}}},
+	})
+	if err == nil || loaded != nil {
+		t.Fatalf("LoadLayered = %#v, %v, want hidden underlay rejection", loaded, err)
+	}
+	loadErr, ok := err.(*LoadError)
+	if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeVersionHookFailed {
+		t.Fatalf("underlay rejection diagnostics = %#v", err)
+	}
+	if len(checked) != 2 || checked[0] != "main-v" || checked[1] != "rejected-peer" {
+		t.Fatalf("checked declarations = %#v", checked)
+	}
+}
+
+func TestSourceVersionUnderlaysRequireActualHeadersButSkipAbsentSources(t *testing.T) {
+	t.Parallel()
+
+	newOptions := func(underlays []Underlay) LayeredOptions[map[string]any] {
+		return LayeredOptions[map[string]any]{
+			Definition: Definition[map[string]any]{Version: &VersionRule{
+				Path:  Keys("meta", "version"),
+				Check: func(Version, *Validation) error { return nil },
+			}},
+			SourceVersionMode: sourceVersionModeRequired,
+			Underlays:         underlays,
+		}
+	}
+	main := Input{Source: Bytes("main", []byte("meta: {version: main-v}\n"))}
+
+	t.Run("required absence", func(t *testing.T) {
+		loaded, err := LoadLayered(main, testSecretBinding(), newOptions([]Underlay{{Inputs: []Input{{Source: Bytes("missing", []byte("value: present\n"))}}}}))
+		if err == nil || loaded != nil {
+			t.Fatalf("LoadLayered = %#v, %v, want missing underlay declaration", loaded, err)
+		}
+		loadErr, ok := err.(*LoadError)
+		if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeVersionError {
+			t.Fatalf("missing underlay diagnostics = %#v", err)
+		}
+	})
+
+	t.Run("empty group and NoSource", func(t *testing.T) {
+		loaded, err := LoadLayered(main, testSecretBinding(), newOptions([]Underlay{{}, {Inputs: []Input{{Source: NoSource()}}}}))
+		if err != nil {
+			t.Fatalf("LoadLayered with absent contributions: %v", err)
+		}
+		if sources := loaded.Inspection.Sources(); len(sources) != 1 {
+			t.Fatalf("empty groups and NoSource created source records: %#v", sources)
+		}
+	})
+}
+
+func TestSourceVersionUnderlayImportsKeepLocalHeadersAndMountedDestinations(t *testing.T) {
+	t.Parallel()
+
+	directory := t.TempDir()
+	underlayPath := writeImportFile(t, directory, "underlay.yml", `meta: {version: peer-root}
+configkit:
+  imports:
+    - path: nested.yml
+      destination: [nested]
+root: retained
+`)
+	writeImportFile(t, directory, "nested.yml", "meta: {version: peer-nested}\nname: nested\n")
+	var checked []string
+	loaded, err := LoadLayered(Input{
+		Source: Bytes("main", []byte("meta: {version: main-v}\n")), Destination: Keys("service"),
+	}, testSecretBinding(), LayeredOptions[map[string]any]{
+		Definition: Definition[map[string]any]{Version: &VersionRule{
+			Path: Keys("meta", "version"),
+			Check: func(version Version, _ *Validation) error {
+				checked = append(checked, version.Text)
+				return nil
+			},
+		}},
+		SourceVersionMode: sourceVersionModeRequired,
+		Underlays: []Underlay{{Inputs: []Input{{
+			Source: File(underlayPath), Imports: true, Destination: Keys("defaults"),
+		}}}},
+	})
+	if err != nil {
+		t.Fatalf("LoadLayered: %v", err)
+	}
+	if len(checked) != 3 {
+		t.Fatalf("checked source declarations = %#v", checked)
+	}
+	for _, want := range []string{"main-v", "peer-root", "peer-nested"} {
+		found := false
+		for _, got := range checked {
+			found = found || got == want
+		}
+		if !found {
+			t.Errorf("checked source declarations %#v do not include %q", checked, want)
+		}
+	}
+	sources := loaded.Inspection.Sources()
+	if len(sources) != 3 || sources[1].Path != filepath.Clean(underlayPath) || sources[2].Destination.String() != "/defaults/nested" {
+		t.Fatalf("underlay and nested import records = %#v", sources)
+	}
+	if sources[1].Layer != LayerUnderlay || sources[1].UnderlayIndex != 1 || sources[2].Layer != LayerUnderlay || sources[2].UnderlayIndex != 1 {
+		t.Fatalf("underlay and nested import tiers = %#v", sources)
+	}
+	defaults, ok := loaded.Config["defaults"].(map[string]any)
+	if !ok || defaults["root"] != "retained" {
+		t.Fatalf("mounted underlay root = %#v", loaded.Config["defaults"])
+	}
+	nested, ok := defaults["nested"].(map[string]any)
+	if !ok || nested["name"] != "nested" {
+		t.Fatalf("mounted nested import = %#v", defaults["nested"])
+	}
+}
+
+func TestSourceVersionUnderlaysProtectRemainingMountedPayload(t *testing.T) {
+	t.Parallel()
+
+	loaded, err := LoadLayered(Input{
+		Source: Bytes("main", []byte("meta: {version: main-v}\n")), Destination: Keys("service"),
+	}, testSecretBinding(), LayeredOptions[map[string]any]{
+		Definition: Definition[map[string]any]{Version: &VersionRule{
+			Path: Keys("meta", "version"), Check: func(Version, *Validation) error { return nil },
+		}},
+		SourceVersionMode: sourceVersionModeRequired,
+		Underlays: []Underlay{{Inputs: []Input{{Source: Bytes("underlay", []byte(
+			"meta: {version: accepted}\nservice: {meta: {version: payload}}\n",
+		))}}}},
+	})
+	if err == nil || loaded != nil {
+		t.Fatalf("LoadLayered = %#v, %v, want protected remaining payload rejection", loaded, err)
+	}
+	loadErr, ok := err.(*LoadError)
+	if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeUnderlayVersion {
+		t.Fatalf("remaining payload diagnostics = %#v", err)
+	}
+}
+
 func TestSourceVersionImportAccountingIncludesConsumedHeaderNodes(t *testing.T) {
 	t.Parallel()