Validate source versions in recursive imports

This commit is contained in:
2026-10-15 15:33:40 +00:00
parent 23ae700194
commit 813d6a38c9
6 changed files with 340 additions and 34 deletions

View File

@@ -45,10 +45,11 @@ type importAccounting struct {
}
type importTraversal struct {
access *filesystemAccess
accounting *importAccounting
limits importLimits
rule *VersionRule
access *filesystemAccess
accounting *importAccounting
limits importLimits
rule *VersionRule
versionState *sourceVersionState
documents []*document.Node
sources []SourceInfo
@@ -62,12 +63,16 @@ func loadImportedDocuments(source Source, context *FilesystemContext, main parse
}
func loadImportedDocumentsTracked(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (*composedDocument, []SourceInfo, []Diagnostic) {
return loadImportedDocumentsTrackedWithVersion(source, context, main, destination, rule, limits, accounting, visited, hostToken, nil)
}
func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
if nativeContext(source, context) {
access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
if diagnostic != nil {
return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
}
return loadImportedWithAccessTracked(access, basePath, mainPath, main, destination, rule, limits, accounting, visited)
return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState)
}
rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
if diagnostic != nil {
@@ -82,7 +87,7 @@ func loadImportedDocumentsTracked(source Source, context *FilesystemContext, mai
if hostToken != nil {
access.token = hostToken
}
return loadImportedWithAccessTracked(access, basePath, mainPath, main, destination, rule, limits, accounting, visited)
return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState)
}
func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
@@ -90,6 +95,10 @@ func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string,
}
func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute) (*composedDocument, []SourceInfo, []Diagnostic) {
return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, nil)
}
func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
_ = access.close()
return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
@@ -118,7 +127,7 @@ func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath
accounting.bytes += main.bytes
accounting.nodes += countInputNodes(main.document)
state := importTraversal{
access: access, accounting: accounting, limits: limits, rule: rule,
access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState,
documents: make([]*document.Node, 0, 1),
sources: []SourceInfo{mainInfo},
active: make(map[resourceIdentity]int), visited: visited,
@@ -139,7 +148,15 @@ func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath
state.active[mainKey] = 0
state.visited[mainKey] = mainRoute
state.stack = append(state.stack, mainRoute)
body, declarations, diagnostics := splitImportDirectives(main.document)
var body *document.Node
var declarations []importDeclaration
var diagnostics []Diagnostic
if versionState != nil && versionState.mode != sourceVersionModeDefault && versionState.mainProcessing != nil {
body = versionState.mainProcessing.payload
declarations = versionState.mainProcessing.imports
} else {
body, declarations, diagnostics = splitImportDirectives(main.document)
}
if len(diagnostics) == 0 {
_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes)
placementNodes := destinationWrapperNodeCount(destination)
@@ -298,9 +315,20 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
}
s.accounting.nodes += nodeCount
body, declarations, diagnostics := splitImportDirectives(root)
if len(diagnostics) != 0 {
return diagnostics
var body *document.Node
var declarations []importDeclaration
var diagnostics []Diagnostic
if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault {
processed, failures := s.versionState.process(root, sourceVersionContribution, destination, sourceID)
if len(failures) != 0 {
return failures
}
body, declarations = processed.payload, processed.imports
} else {
body, declarations, diagnostics = splitImportDirectives(root)
if len(diagnostics) != 0 {
return diagnostics
}
}
_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes)
if err != nil {
@@ -311,10 +339,14 @@ func (s *importTraversal) visitPath(basePath string, declaration importDeclarati
return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
}
s.accounting.nodes += placementNodes
if versionNode := fragmentVersionClaim(placed, s.rule); versionNode != nil {
protectionRule := s.rule
if s.versionState != nil && s.versionState.effectiveRule != nil {
protectionRule = s.versionState.effectiveRule
}
if versionNode := fragmentVersionClaim(placed, protectionRule); versionNode != nil {
versionLocation := publicLocation(versionNode.Location)
return []Diagnostic{{
Code: CodeImportVersion, Severity: SeverityError, Path: s.rule.Path.clone(),
Code: CodeImportVersion, Severity: SeverityError, Path: protectionRule.Path.clone(),
Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation,
}}
}

View File

@@ -170,9 +170,12 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
return nil, accumulator.loadError()
}
}
input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, request.definition.Version, request.sourceVersionMode)
versionState := &sourceVersionState{
localRule: request.definition.Version, effectiveRule: request.protectionVersion(), mode: request.sourceVersionMode,
}
input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, versionState)
accumulator.sources = input.sources
accumulator.add(input.versionDiagnostics...)
accumulator.add(versionState.diagnostics...)
accumulator.add(diagnostics...)
if hasErrors(diagnostics) {
return nil, accumulator.loadError()

View File

@@ -8,14 +8,13 @@ import (
// assembledInput is the source/document handoff across schema and version checks.
type assembledInput struct {
parsed parsedSource
document *document.Node
composition *composedDocument
inheritance *inheritanceScope
sources []SourceInfo
contexts map[string]ResourceContext
versionEvidence *sourceVersionEvidence
versionDiagnostics []Diagnostic
parsed parsedSource
document *document.Node
composition *composedDocument
inheritance *inheritanceScope
sources []SourceInfo
contexts map[string]ResourceContext
versionState *sourceVersionState
}
func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput, []Diagnostic) {
@@ -23,10 +22,10 @@ func assembleBaseInput[T any](source Source, options Options[T]) (assembledInput
}
func assembleBaseInputTracked[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (assembledInput, []Diagnostic) {
return assembleBaseInputTrackedWithVersion(source, options, destination, accounting, visited, hostToken, nil, sourceVersionModeDefault)
return assembleBaseInputTrackedWithVersion(source, options, destination, accounting, visited, hostToken, nil)
}
func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, localVersion *VersionRule, mode sourceVersionMode) (assembledInput, []Diagnostic) {
func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (assembledInput, []Diagnostic) {
limits := defaultImportLimits()
parseLimits := defaultParseLimits()
parseLimits.source.remaining = limits.bytes - accounting.bytes
@@ -34,7 +33,7 @@ func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T
if parsed.source != nil {
parsed.source.Destination = destination.clone()
}
input := assembledInput{parsed: parsed, document: parsed.document}
input := assembledInput{parsed: parsed, document: parsed.document, versionState: versionState}
if len(diagnostics) != 0 {
input.sources = sourceInfos(parsed.source)
// Mapping value issues already identify application content. Other early
@@ -48,16 +47,19 @@ func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T
}
return input, diagnostics
}
if mode != sourceVersionModeDefault {
processed := processSourceVersion(parsed.document, localVersion, sourceVersionMain, destination, mode)
input.versionEvidence = processed.evidence
input.versionDiagnostics = processed.diagnostics
if hasErrors(processed.diagnostics) {
return input, processed.diagnostics
if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
processed, failures := versionState.process(parsed.document, sourceVersionMain, destination, sourceID)
if len(failures) != 0 {
return input, failures
}
versionState.mainProcessing = &processed
}
if options.Imports {
input.composition, input.sources, diagnostics = loadImportedDocumentsTracked(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken)
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithVersion(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState)
if len(diagnostics) != 0 {
return input, diagnostics
}

View File

@@ -23,6 +23,7 @@ type sourceVersionAncestor struct {
}
type sourceVersionEvidence struct {
sourceID string
path Path
resolvedPath Path
destination Path
@@ -34,6 +35,35 @@ type sourceVersionEvidence struct {
redacted bool
}
type sourceVersionState struct {
localRule *VersionRule
effectiveRule *VersionRule
mode sourceVersionMode
mainProcessing *sourceVersionProcessingResult
evidence []*sourceVersionEvidence
diagnostics []Diagnostic
}
func (state *sourceVersionState) process(root *document.Node, role sourceVersionRole, destination Path, sourceID string) (sourceVersionProcessingResult, []Diagnostic) {
if state == nil || state.mode == sourceVersionModeDefault {
return sourceVersionProcessingResult{}, nil
}
result := processSourceVersion(root, state.localRule, role, destination, state.mode)
if result.evidence != nil {
result.evidence.sourceID = sourceID
state.evidence = append(state.evidence, result.evidence)
}
var failures []Diagnostic
for _, diagnostic := range result.diagnostics {
if diagnostic.Severity == SeverityWarning {
state.diagnostics = append(state.diagnostics, diagnostic)
} else if diagnostic.Severity == SeverityError {
failures = append(failures, diagnostic)
}
}
return result, failures
}
type sourceVersionProcessingResult struct {
payload *document.Node
imports []importDeclaration

View File

@@ -19,6 +19,17 @@ type sourceVersionNullConfig struct {
Version *string `yaml:"version"`
}
type sourceVersionImportGroup struct {
Name string `yaml:"name"`
}
type sourceVersionImportConfig struct {
Service struct {
Version string `yaml:"version"`
Groups map[string]sourceVersionImportGroup `yaml:"groups"`
} `yaml:"service"`
}
func sourceVersionTestDocument(t *testing.T, input string) *document.Node {
t.Helper()
parsed, diagnostics := parseSource(Bytes("source-version.yml", []byte(input)), "source-version", defaultParseLimits())
@@ -592,6 +603,213 @@ func TestSourceVersionLoadRejectsOptionsBeforeReadingMain(t *testing.T) {
requireSourceVersionDiagnostic(t, loadErr.Diagnostics(), CodeOptionsInvalid, "")
}
func TestSourceVersionImportsCheckNestedAndGlobSourcesBeforeComposition(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `version: main
configkit:
imports:
- glob: fragments/*.yml
destination: [groups]
`)
firstPath := writeImportFile(t, directory, "fragments/a.yml", `version: first
configkit:
imports:
- path: nested/child.yml
destination: [nested]
a: {name: A}
`)
nestedPath := writeImportFile(t, directory, "fragments/nested/child.yml", "version: nested\nname: N\n")
secondPath := writeImportFile(t, directory, "fragments/b.yml", "version: second\nb: {name: B}\n")
checked := make(map[string]int)
loaded, err := LoadLayered(Input{
Source: File(mainPath), Imports: true, Destination: Keys("service"),
}, testSecretBinding(), LayeredOptions[sourceVersionImportConfig]{
SourceVersionMode: sourceVersionModeRequired,
Definition: Definition[sourceVersionImportConfig]{Version: &VersionRule{
Path: Keys("version"),
Check: func(version Version, validation *Validation) error {
checked[version.Text]++
validation.Warning(Path{}, "accepted")
return nil
},
}},
})
if err != nil || loaded == nil {
var loadErr *LoadError
if errors.As(err, &loadErr) {
t.Fatalf("source-local import load = %#v, diagnostics %#v", loaded, loadErr.Diagnostics())
}
t.Fatalf("source-local import load = %#v, %v", loaded, err)
}
if loaded.Config.Service.Version != "main" || loaded.Config.Service.Groups["a"].Name != "A" ||
loaded.Config.Service.Groups["b"].Name != "B" || loaded.Config.Service.Groups["nested"].Name != "N" {
t.Fatalf("mounted source-local imports = %#v", loaded.Config.Service)
}
if len(checked) != 4 || checked["main"] != 1 || checked["first"] != 1 || checked["nested"] != 1 || checked["second"] != 1 {
t.Fatalf("checked declarations = %#v, want each independent source once", checked)
}
warnings := testLoaded(loaded).Diagnostics()
if len(warnings) != 4 {
t.Fatalf("accepted source warnings = %#v, want one per checked source", warnings)
}
for _, warning := range warnings {
if warning.Code != CodeVersionWarning || warning.Severity != SeverityWarning {
t.Fatalf("accepted source warning = %#v", warning)
}
}
wantDestinations := map[string]string{
canonicalTestPath(t, mainPath): "/service",
canonicalTestPath(t, firstPath): "/service/groups",
canonicalTestPath(t, secondPath): "/service/groups",
canonicalTestPath(t, nestedPath): "/service/groups/nested",
}
for _, source := range testLoaded(loaded).Sources() {
want, found := wantDestinations[source.Path]
if !found || source.Destination.String() != want {
t.Errorf("source %q destination = %q, want %q", source.Path, source.Destination.String(), want)
}
delete(wantDestinations, source.Path)
}
if len(wantDestinations) != 0 {
t.Errorf("sources not retained: %#v", wantDestinations)
}
}
func TestSourceVersionImportsRejectHiddenAndMissingDeclarations(t *testing.T) {
t.Parallel()
t.Run("rejected child is not hidden by the main declaration", func(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "version: main\nconfigkit: {imports: [{path: child.yml}]}\n")
writeImportFile(t, directory, "child.yml", "version: rejected\n")
checks := 0
_, err := Load(File(mainPath), testSecretBinding(), Options[sourceVersionNullConfig]{
Imports: true, SourceVersionMode: sourceVersionModeIfPresent,
Definition: Definition[sourceVersionNullConfig]{Version: &VersionRule{
Path: Keys("version"), Check: func(version Version, _ *Validation) error {
checks++
if version.Text == "rejected" {
return errors.New("rejected")
}
return nil
},
}},
})
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("rejected child error = %T %v, want *LoadError", err, err)
}
diagnostic := requireSourceVersionDiagnostic(t, loadErr.Diagnostics(), CodeVersionHookFailed, "/version")
if diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" || checks != 2 {
t.Fatalf("rejected child diagnostic/callbacks = %#v, %d", diagnostic, checks)
}
})
t.Run("required child omission reports its nearest source location", func(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "version: main\nconfigkit: {imports: [{path: child.yml}]}\n")
writeImportFile(t, directory, "child.yml", "name: child\n")
checks := 0
_, err := Load(File(mainPath), testSecretBinding(), Options[sourceVersionNullConfig]{
Imports: true, SourceVersionMode: sourceVersionModeRequired,
Definition: Definition[sourceVersionNullConfig]{Version: &VersionRule{
Path: Keys("version"), Check: func(Version, *Validation) error { checks++; return nil },
}},
})
var loadErr *LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("required child omission = %T %v, want *LoadError", err, err)
}
diagnostic := requireSourceVersionDiagnostic(t, loadErr.Diagnostics(), CodeVersionError, "/version")
if diagnostic.Primary == nil || diagnostic.Primary.SourceID != "source-2" || diagnostic.Primary.Line != 1 || checks != 1 {
t.Fatalf("required omission diagnostic/callbacks = %#v, %d", diagnostic, checks)
}
})
t.Run("optional missing child remains absent", func(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", `version: main
configkit:
imports:
- path: missing.yml
optional: true
`)
checks := 0
loaded, err := Load(File(mainPath), testSecretBinding(), Options[sourceVersionNullConfig]{
Imports: true, SourceVersionMode: sourceVersionModeRequired,
Definition: Definition[sourceVersionNullConfig]{Version: &VersionRule{
Path: Keys("version"), Check: func(Version, *Validation) error { checks++; return nil },
}},
})
if err != nil || loaded == nil || checks != 1 || len(testLoaded(loaded).Sources()) != 1 {
t.Fatalf("optional missing source = %#v, checks %d, %v", loaded, checks, err)
}
})
}
func TestSourceVersionImportAccountingIncludesConsumedHeaderNodes(t *testing.T) {
t.Parallel()
directory := t.TempDir()
mainPath := writeImportFile(t, directory, "main.yml", "configkit: {imports: [{path: child.yml}]}\n")
childPath := writeImportFile(t, directory, "child.yml", "version: accepted\nname: child\n")
main := parseImportMain(t, mainPath)
child, diagnostics := parseSource(File(childPath), "source-child", defaultParseLimits())
if len(diagnostics) != 0 {
t.Fatalf("child parse diagnostics = %#v", diagnostics)
}
rule := &VersionRule{Path: Keys("version"), Check: func(Version, *Validation) error { return nil }}
state := &sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired}
wantNodes := countInputNodes(main.document) + countInputNodes(child.document)
wantBytes := main.bytes + child.bytes
limits := defaultImportLimits()
limits.bytes = wantBytes
limits.nodes = wantNodes
accounting := &importAccounting{}
composed, sources, foundDiagnostics := loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, limits, accounting, nil, nil, state,
)
if len(foundDiagnostics) != 0 || composed == nil || len(sources) != 2 || accounting.nodes != wantNodes || accounting.bytes != wantBytes {
t.Fatalf("exact raw byte/node accounting = %#v, %d/%d, want %d/%d, %#v", composed, accounting.bytes, accounting.nodes, wantBytes, wantNodes, foundDiagnostics)
}
if len(state.evidence) != 1 || state.evidence[0].sourceID != "source-2" || !state.evidence[0].checked || state.evidence[0].version.Text != "accepted" {
t.Fatalf("acquired child evidence = %#v", state.evidence)
}
if _, present := documentAtPath(composed.root, Keys("version")); present {
t.Fatalf("consumed child header remains in application document: %#v", composed.root)
}
below := defaultImportLimits()
below.nodes = wantNodes - 1
accounting = &importAccounting{}
composed, _, foundDiagnostics = loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, below, accounting, nil, nil,
&sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired},
)
if composed != nil || len(foundDiagnostics) != 1 || foundDiagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("raw header nodes were not charged: %#v, %#v", composed, foundDiagnostics)
}
below = defaultImportLimits()
below.bytes = wantBytes - 1
accounting = &importAccounting{}
composed, _, foundDiagnostics = loadImportedDocumentsTrackedWithVersion(
File(mainPath), nil, main, Path{}, rule, below, accounting, nil, nil,
&sourceVersionState{localRule: rule, effectiveRule: rule, mode: sourceVersionModeRequired},
)
if composed != nil || len(foundDiagnostics) != 1 || foundDiagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("raw header bytes were not charged: %#v, %#v", composed, foundDiagnostics)
}
}
func mustParseSourceVersionPath(t *testing.T, input string) Path {
t.Helper()
path, err := ParsePath(input)