Validate source versions in recursive imports

This commit is contained in:
2026-10-15 15:33:40 +00:00
parent 23ae700194
commit 813d6a38c9
6 changed files with 340 additions and 34 deletions

View File

@@ -1,6 +1,6 @@
# Implement shared source version checks
Status: Stages 1–4 complete; recursive import integration has not started.
Status: Stages 1–5 complete; direct underlay integration has not started.
Audience: a Luna coding agent implementing one numbered stage per prompt, in
order. The [feature roadmap](source-version-checks.md) owns target behavior and
@@ -343,6 +343,27 @@ The engine mode remains private; public options are scheduled separately.
Validate: import/glob/destination/accounting/version tests and everyday check.
Done: imports use one policy without lost warnings or changed physical traversal.
Completion evidence: threaded one per-load source-version state through the
main import entry and recursive traversal. Each parsed child is checked against
its source-local rule before placement; accepted headers are removed from an
owned payload while the original root still supplies raw byte/node charges and
the processor's original import declarations drive nested traversal. Effective
main ownership is checked against the remaining placed payload. Evidence is
retained with the acquired SourceID, and warning diagnostics accumulate without
stopping traversal. Existing identity, cycle, optional-source, source-record,
resource-context, and accounting paths remain in use. Added coverage for nested
and glob imports at mounted relative destinations, accepted unequal declarations,
rejected hidden declarations, required omissions with source-local locations,
optional missing imports, evidence identity, and raw header byte/node charges. The
existing default import-version failure coverage also passes. These checks
passed with Go `go1.26.8 linux/amd64`:
- `GOTOOLCHAIN=local GOPROXY=off go test ./internal/engine -run 'SourceVersion|Version|Import|Glob|Destination' -count=1`
- `GOTOOLCHAIN=local ./scripts/check.sh`
- `git diff --check`
No environmental limitation affected validation; checks ran on Linux/amd64.
## Stage 6 — Integrate direct underlays and their strict groups
1. Thread state through acquireUnderlayGroup/grouped assembly. Direct underlays

View File

@@ -49,6 +49,7 @@ type importTraversal struct {
accounting *importAccounting
limits importLimits
rule *VersionRule
versionState *sourceVersionState
documents []*document.Node
sources []SourceInfo
@@ -62,12 +63,16 @@ func loadImportedDocuments(source Source, context *FilesystemContext, main parse
}
func loadImportedDocumentsTracked(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (*composedDocument, []SourceInfo, []Diagnostic) {
return loadImportedDocumentsTrackedWithVersion(source, context, main, destination, rule, limits, accounting, visited, hostToken, nil)
}
func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
if nativeContext(source, context) {
access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
if diagnostic != nil {
return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
}
return loadImportedWithAccessTracked(access, basePath, mainPath, main, destination, rule, limits, accounting, visited)
return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState)
}
rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
if diagnostic != nil {
@@ -82,7 +87,7 @@ func loadImportedDocumentsTracked(source Source, context *FilesystemContext, mai
if hostToken != nil {
access.token = hostToken
}
return loadImportedWithAccessTracked(access, basePath, mainPath, main, destination, rule, limits, accounting, visited)
return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState)
}
func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
@@ -90,6 +95,10 @@ func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string,
}
func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute) (*composedDocument, []SourceInfo, []Diagnostic) {
return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, nil)
}
func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
_ = access.close()
return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
@@ -118,7 +127,7 @@ func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath
accounting.bytes += main.bytes
accounting.nodes += countInputNodes(main.document)
state := importTraversal{
access: access, accounting: accounting, limits: limits, rule: rule,
access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState,
documents: make([]*document.Node, 0, 1),
sources: []SourceInfo{mainInfo},
active: make(map[resourceIdentity]int), visited: visited,
@@ -139,7 +148,15 @@ func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath
state.active[mainKey] = 0
state.visited[mainKey] = mainRoute
state.stack = append(state.stack, mainRoute)
body, declarations, diagnostics := splitImportDirectives(main.document)
var body *document.Node
var declarations []importDeclaration
var diagnostics []Diagnostic
if versionState != nil && versionState.mode != sourceVersionModeDefault && versionState.mainProcessing != nil {
body = versionState.mainProcessing.payload
declarations = versionState.mainProcessing.imports
} else {
body, declarations, diagnostics = splitImportDirectives(main.document)
}
if len(diagnostics) == 0 {
_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes)
placementNodes := destinationWrapperNodeCount(destination)
@@ -298,10 +315,21 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
}
s.accounting.nodes += nodeCount
body, declarations, diagnostics := splitImportDirectives(root)
var body *document.Node
var declarations []importDeclaration
var diagnostics []Diagnostic
if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault {
processed, failures := s.versionState.process(root, sourceVersionContribution, destination, sourceID)
if len(failures) != 0 {
return failures
}
body, declarations = processed.payload, processed.imports
} else {
body, declarations, diagnostics = splitImportDirectives(root)
if len(diagnostics) != 0 {
return diagnostics
}
}
_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)}
@@ -311,10 +339,14 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
}
s.accounting.nodes += placementNodes
if versionNode := fragmentVersionClaim(placed, s.rule); versionNode != nil {
protectionRule := s.rule
if s.versionState != nil && s.versionState.effectiveRule != nil {
protectionRule = s.versionState.effectiveRule
}
if versionNode := fragmentVersionClaim(placed, protectionRule); versionNode != nil {
versionLocation := publicLocation(versionNode.Location)
return []Diagnostic{{
Code: CodeImportVersion, Severity: SeverityError, Path: s.rule.Path.clone(),
Code: CodeImportVersion, Severity: SeverityError, Path: protectionRule.Path.clone(),
Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation,
}}
}

View File

@@ -170,9 +170,12 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
return nil, accumulator.loadError()
}
}
input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, request.definition.Version, request.sourceVersionMode)
versionState := &sourceVersionState{
localRule: request.definition.Version, effectiveRule: request.protectionVersion(), mode: request.sourceVersionMode,
}
input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, versionState)
accumulator.sources = input.sources
accumulator.add(input.versionDiagnostics...)
accumulator.add(versionState.diagnostics...)
accumulator.add(diagnostics...)
if hasErrors(diagnostics) {
return nil, accumulator.loadError()

View File

@@ -14,8 +14,7 @@ type assembledInput struct {
inheritance *inheritanceScope
sources []SourceInfo
contexts map[string]ResourceContext
versionEvidence *sourceVersionEvidence
versionDiagnostics []Diagnostic
versionState *sourceVersionState
}
func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput, []Diagnostic) {
@@ -23,10 +22,10 @@ func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput
}
func assembleBaseInputTracked[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (assembledInput, []Diagnostic) {
return assembleBaseInputTrackedWithVersion(source, options, destination, accounting, visited, hostToken, nil, sourceVersionModeDefault)
return assembleBaseInputTrackedWithVersion(source, options, destination, accounting, visited, hostToken, nil)
}
func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, localVersion *VersionRule, mode sourceVersionMode) (assembledInput, []Diagnostic) {
func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (assembledInput, []Diagnostic) {
limits := defaultImportLimits()
parseLimits := defaultParseLimits()
parseLimits.source.remaining = limits.bytes - accounting.bytes
@@ -34,7 +33,7 @@ func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T
if parsed.source != nil {
parsed.source.Destination = destination.clone()
}
input := assembledInput{parsed: parsed, document: parsed.document}
input := assembledInput{parsed: parsed, document: parsed.document, versionState: versionState}
if len(diagnostics) != 0 {
input.sources = sourceInfos(parsed.source)
// Mapping value issues already identify application content. Other early
@@ -48,16 +47,19 @@ func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T
}
return input, diagnostics
}
if mode != sourceVersionModeDefault {
processed := processSourceVersion(parsed.document, localVersion, sourceVersionMain, destination, mode)
input.versionEvidence = processed.evidence
input.versionDiagnostics = processed.diagnostics
if hasErrors(processed.diagnostics) {
return input, processed.diagnostics
if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
processed, failures := versionState.process(parsed.document, sourceVersionMain, destination, sourceID)
if len(failures) != 0 {
return input, failures
}
versionState.mainProcessing = &processed
}
if options.Imports {
input.composition, input.sources, diagnostics = loadImportedDocumentsTracked(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken)
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithVersion(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState)
if len(diagnostics) != 0 {
return input, diagnostics
}

View File

@@ -23,6 +23,7 @@ type sourceVersionAncestor struct {
}
type sourceVersionEvidence struct {
sourceID string
path Path
resolvedPath Path
destination Path
@@ -34,6 +35,35 @@ type sourceVersionEvidence struct {
redacted bool
}
type sourceVersionState struct {
localRule *VersionRule
effectiveRule *VersionRule
mode sourceVersionMode
mainProcessing *sourceVersionProcessingResult
evidence []*sourceVersionEvidence
diagnostics []Diagnostic
}
func (state *sourceVersionState) process(root *document.Node, role sourceVersionRole, destination Path, sourceID string) (sourceVersionProcessingResult, []Diagnostic) {
if state == nil || state.mode == sourceVersionModeDefault {
return sourceVersionProcessingResult{}, nil
}
result := processSourceVersion(root, state.localRule, role, destination, state.mode)
if result.evidence != nil {
result.evidence.sourceID = sourceID
state.evidence = append(state.evidence, result.evidence)
}
var failures []Diagnostic
for _, diagnostic := range result.diagnostics {
if diagnostic.Severity == SeverityWarning {
state.diagnostics = append(state.diagnostics, diagnostic)
} else if diagnostic.Severity == SeverityError {
failures = append(failures, diagnostic)
}
}
return result, failures
}
type sourceVersionProcessingResult struct {
payload *document.Node
imports []importDeclaration

View File

@@ -19,6 +19,17 @@ type sourceVersionNullConfig struct {
Version *string `yaml:"version"`
}
type sourceVersionImportGroup struct {
Name string `yaml:"name"`
}
type sourceVersionImportConfig struct {
Service struct {
Version string `yaml:"version"`
Groups map[string]sourceVersionImportGroup `yaml:"groups"`
} `yaml:"service"`
}
func sourceVersionTestDocument(t *testing.T, input string) *document.Node {
t.Helper()
parsed, diagnostics := parseSource(Bytes("source-version.yml", []byte(input)), "source-version", defaultParseLimits())
@@ -592,6 +603,213 @@ func TestSourceVersionLoadRejectsOptionsBeforeReadingMain(t *testing.T) {
requireSourceVersionDiagnostic(t, loadErr.Diagnostics(), CodeOptionsInvalid, "")
}
func TestSourceVersionImportsCheckNestedAndGlobSourcesBeforeComposition(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `version: main
configkit:
imports:
- glob: fragments/*.yml
destination: [groups]
`)
firstPath := writeImportFile(t, directory, "fragments/a.yml", `version: first
configkit:
imports:
- path: nested/child.yml
destination: [nested]
a: {name: A}
`)
nestedPath := writeImportFile(t, directory, "fragments/nested/child.yml", "version: nested\nname: N\n")
secondPath := writeImportFile(t, directory, "fragments/b.yml", "version: second\nb: {name: B}\n")
checked := make(map[string]int)
loaded, err := LoadLayered(Input{
Source: File(mainPath), Imports: true, Destination: Keys("service"),
}, testSecretBinding(), LayeredOptions[sourceVersionImportConfig]{
SourceVersionMode: sourceVersionModeRequired,
Definition: Definition[sourceVersionImportConfig]{Version: &VersionRule{
Path: Keys("version"),
Check: func(version Version, validation *Validation) error {
checked[version.Text]++
validation.Warning(Path{}, "accepted")
return nil
},
}},
})
if err != nil || loaded == nil {
var loadErr *LoadError
if errors.As(err, &loadErr) {
t.Fatalf("source-local import load = %#v, diagnostics %#v", loaded, loadErr.Diagnostics())
}
t.Fatalf("source-local import load = %#v, %v", loaded, err)
}
if loaded.Config.Service.Version != "main" || loaded.Config.Service.Groups["a"].Name != "A" ||
loaded.Config.Service.Groups["b"].Name != "B" || loaded.Config.Service.Groups["nested"].Name != "N" {
t.Fatalf("mounted source-local imports = %#v", loaded.Config.Service)
}
if len(checked) != 4 || checked["main"] != 1 || checked["first"] != 1 || checked["nested"] != 1 || checked["second"] != 1 {
t.Fatalf("checked declarations = %#v, want each independent source once", checked)
}
warnings := testLoaded(loaded).Diagnostics()
if len(warnings) != 4 {
t.Fatalf("accepted source warnings = %#v, want one per checked source", warnings)
}
for _, warning := range warnings {
if warning.Code != CodeVersionWarning || warning.Severity != SeverityWarning {
t.Fatalf("accepted source warning = %#v", warning)
}
}
wantDestinations := map[string]string{
canonicalTestPath(t, mainPath): "/service",
canonicalTestPath(t, firstPath): "/service/groups",
canonicalTestPath(t, secondPath): "/service/groups",
canonicalTestPath(t, nestedPath): "/service/groups/nested",
}
for _, source := range testLoaded(loaded).Sources() {
want, found := wantDestinations[source.Path]
if !found || source.Destination.String() != want {
t.Errorf("source %q destination = %q, want %q", source.Path, source.Destination.String(), want)
}
delete(wantDestinations, source.Path)
}
if len(wantDestinations) != 0 {
t.Errorf("sources not retained: %#v", wantDestinations)
}
}
func TestSourceVersionImportsRejectHiddenAndMissingDeclarations(t *testing.T) {
t.Parallel()
t.Run("rejected child is not hidden by the main declaration", func(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "version: main\nconfigkit: {imports: [{path: child.yml}]}\n")
writeImportFile(t, directory, "child.yml", "version: rejected\n")
checks := 0
_, err := Load(File(mainPath), testSecretBinding(), Options[sourceVersionNullConfig]{
Imports: true, SourceVersionMode: sourceVersionModeIfPresent,
Definition: Definition[sourceVersionNullConfig]{Version: &VersionRule{
Path: Keys("version"), Check: func(version Version, _ *Validation) error {
checks++
if version.Text == "rejected" {
return errors.New("rejected")
}
return nil
},
}},
})
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("rejected child error = %T %v, want *LoadError", err, err)
}
diagnostic := requireSourceVersionDiagnostic(t, loadErr.Diagnostics(), CodeVersionHookFailed, "/version")
if diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" || checks != 2 {
t.Fatalf("rejected child diagnostic/callbacks = %#v, %d", diagnostic, checks)
}
})
t.Run("required child omission reports its nearest source location", func(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "version: main\nconfigkit: {imports: [{path: child.yml}]}\n")
writeImportFile(t, directory, "child.yml", "name: child\n")
checks := 0
_, err := Load(File(mainPath), testSecretBinding(), Options[sourceVersionNullConfig]{
Imports: true, SourceVersionMode: sourceVersionModeRequired,
Definition: Definition[sourceVersionNullConfig]{Version: &VersionRule{
Path: Keys("version"), Check: func(Version, *Validation) error { checks++; return nil },
}},
})
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("required child omission = %T %v, want *LoadError", err, err)
}
diagnostic := requireSourceVersionDiagnostic(t, loadErr.Diagnostics(), CodeVersionError, "/version")
if diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" || diagnostic.Primary.Line != 1 || checks != 1 {
t.Fatalf("required omission diagnostic/callbacks = %#v, %d", diagnostic, checks)
}
})
t.Run("optional missing child remains absent", func(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `version: main
configkit:
imports:
- path: missing.yml
optional: true
`)
checks := 0
loaded, err := Load(File(mainPath), testSecretBinding(), Options[sourceVersionNullConfig]{
Imports: true, SourceVersionMode: sourceVersionModeRequired,
Definition: Definition[sourceVersionNullConfig]{Version: &VersionRule{
Path: Keys("version"), Check: func(Version, *Validation) error { checks++; return nil },
}},
})
if err != nil || loaded == nil || checks != 1 || len(testLoaded(loaded).Sources()) != 1 {
t.Fatalf("optional missing source = %#v, checks %d, %v", loaded, checks, err)
}
})
}
func TestSourceVersionImportAccountingIncludesConsumedHeaderNodes(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: child.yml}]}\n")
childPath := writeImportFile(t, directory, "child.yml", "version: accepted\nname: child\n")
main := parseImportMain(t, mainPath)
child, diagnostics := parseSource(File(childPath), "source-child", defaultParseLimits())
if len(diagnostics) != 0 {
t.Fatalf("child parse diagnostics = %#v", diagnostics)
}
rule := &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }}
state := &sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired}
wantNodes := countInputNodes(main.document) + countInputNodes(child.document)
wantBytes := main.bytes + child.bytes
limits := defaultImportLimits()
limits.bytes = wantBytes
limits.nodes = wantNodes
accounting := &importAccounting{}
composed, sources, foundDiagnostics := loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, limits, accounting, nil, nil, state,
)
if len(foundDiagnostics) != 0 || composed == nil || len(sources) != 2 || accounting.nodes != wantNodes || accounting.bytes != wantBytes {
t.Fatalf("exact raw byte/node accounting = %#v, %d/%d, want %d/%d, %#v", composed, accounting.bytes, accounting.nodes, wantBytes, wantNodes, foundDiagnostics)
}
if len(state.evidence) != 1 || state.evidence[0].sourceID != "source-2" || !state.evidence[0].checked || state.evidence[0].version.Text != "accepted" {
t.Fatalf("acquired child evidence = %#v", state.evidence)
}
if _, present := documentAtPath(composed.root, Keys("version")); present {
t.Fatalf("consumed child header remains in application document: %#v", composed.root)
}
below := defaultImportLimits()
below.nodes = wantNodes - 1
accounting = &importAccounting{}
composed, _, foundDiagnostics = loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, below, accounting, nil, nil,
&sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired},
)
if composed != nil || len(foundDiagnostics) != 1 || foundDiagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("raw header nodes were not charged: %#v, %#v", composed, foundDiagnostics)
}
below = defaultImportLimits()
below.bytes = wantBytes - 1
accounting = &importAccounting{}
composed, _, foundDiagnostics = loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, below, accounting, nil, nil,
&sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired},
)
if composed != nil || len(foundDiagnostics) != 1 || foundDiagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("raw header bytes were not charged: %#v, %#v", composed, foundDiagnostics)
}
}
func mustParseSourceVersionPath(t *testing.T, input string) Path {
t.Helper()
path, err := ParsePath(input)