eric/ConfigKit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Check source versions in direct underlays

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-15 15:40:11 +00:00
parent 813d6a38c9
commit 184f29c7af
6 changed files with 284 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
internal/engine/imports.go
View File

@@ -67,12 +67,20 @@ func loadImportedDocumentsTracked(source Source, context *FilesystemContext, mai
}
func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
var processing *sourceVersionProcessingResult
if versionState != nil && versionState.mode != sourceVersionModeDefault {
processing = versionState.mainProcessing
}
return loadImportedDocumentsTrackedWithProcessing(source, context, main, destination, rule, limits, accounting, visited, hostToken, versionState, processing)
}
func loadImportedDocumentsTrackedWithProcessing(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) {
if nativeContext(source, context) {
access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
if diagnostic != nil {
return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
}
return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState)
return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
}
rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
if diagnostic != nil {
@@ -87,7 +95,7 @@ func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemC
if hostToken != nil {
access.token = hostToken
}
return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState)
return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
}
func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
@@ -99,6 +107,14 @@ func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath
}
func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
var processing *sourceVersionProcessingResult
if versionState != nil && versionState.mode != sourceVersionModeDefault {
processing = versionState.mainProcessing
}
return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
}
func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) {
if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
_ = access.close()
return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
@@ -151,9 +167,9 @@ func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath
var body *document.Node
var declarations []importDeclaration
var diagnostics []Diagnostic
if versionState != nil && versionState.mode != sourceVersionModeDefault && versionState.mainProcessing != nil {
body = versionState.mainProcessing.payload
declarations = versionState.mainProcessing.imports
if processing != nil {
body = processing.payload
declarations = processing.imports
} else {
body, declarations, diagnostics = splitImportDirectives(main.document)
}

2
internal/engine/load.go
View File

@@ -176,6 +176,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
input, diagnostics := assembleBaseInputTrackedWithVersion(request.main.source, mainOptions, request.main.destination, accounting, nil, nil, versionState)
accumulator.sources = input.sources
accumulator.add(versionState.diagnostics...)
reportedVersionDiagnostics := len(versionState.diagnostics)
accumulator.add(diagnostics...)
if hasErrors(diagnostics) {
return nil, accumulator.loadError()
@@ -200,6 +201,7 @@ func executeLoadRequest[T any](request loadRequest[T], secrets SecretBinding, lo
} else {
input, diagnostics = assembleLayeredInput(request, input, accounting)
}
accumulator.add(versionState.diagnostics[reportedVersionDiagnostics:]...)
accumulator.sources = input.sources
accumulator.add(diagnostics...)
if len(diagnostics) != 0 {

17
internal/engine/load_input.go
View File

@@ -26,6 +26,10 @@ func assembleBaseInputTracked[T any](source Source, options Options[T], destinat
}
func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (assembledInput, []Diagnostic) {
return assembleBaseInputTrackedForRole(source, options, destination, accounting, visited, hostToken, versionState, sourceVersionMain)
}
func assembleBaseInputTrackedForRole[T any](source Source, options Options[T], destination Path, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, role sourceVersionRole) (assembledInput, []Diagnostic) {
limits := defaultImportLimits()
parseLimits := defaultParseLimits()
parseLimits.source.remaining = limits.bytes - accounting.bytes
@@ -47,25 +51,32 @@ func assembleBaseInputTrackedWithVersion[T any](source Source, options Options[T
}
return input, diagnostics
}
var versionProcessing *sourceVersionProcessingResult
if versionState != nil && versionState.mode != sourceVersionModeDefault {
var sourceID string
if parsed.source != nil {
sourceID = parsed.source.ID
}
processed, failures := versionState.process(parsed.document, sourceVersionMain, destination, sourceID)
processed, failures := versionState.process(parsed.document, role, destination, sourceID)
if len(failures) != 0 {
return input, failures
}
versionState.mainProcessing = &processed
versionProcessing = &processed
if role == sourceVersionMain {
versionState.mainProcessing = &processed
}
}
if options.Imports {
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithVersion(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState)
input.composition, input.sources, diagnostics = loadImportedDocumentsTrackedWithProcessing(source, options.Filesystem, parsed, destination, options.Definition.Version, limits, accounting, visited, hostToken, versionState, versionProcessing)
if len(diagnostics) != 0 {
return input, diagnostics
}
input.document = input.composition.root
} else {
input.sources = sourceInfos(parsed.source)
if versionProcessing != nil {
input.document = versionProcessing.payload
}
if options.Filesystem != nil && !activeOverlay(options.Overlay) {
if diagnostic := validateFilesystemContext(source, options.Filesystem, parsed); diagnostic != nil {
return input, []Diagnostic{*diagnostic}

8
internal/engine/load_request.go
View File

@@ -79,7 +79,7 @@ func loadGroupedWithSecretLookup[T any](request loadRequest[T], secrets SecretBi
func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, accounting *importAccounting) (assembledInput, []Diagnostic) {
groups := make([]*composedDocument, 0, len(request.underlays))
for index, group := range request.underlays {
composition, sources, contexts, diagnostics := acquireUnderlayGroup(group, index+1, request.protectionVersion(), accounting)
composition, sources, contexts, diagnostics := acquireUnderlayGroupWithVersion(group, index+1, request.protectionVersion(), accounting, input.versionState)
input.sources = append(input.sources, sources...)
mergeResourceContexts(&input.contexts, contexts)
if len(diagnostics) != 0 {
@@ -149,6 +149,10 @@ func assembleLayeredInput[T any](request loadRequest[T], input assembledInput, a
}
func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, accounting *importAccounting) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
return acquireUnderlayGroupWithVersion(group, index, rule, accounting, nil)
}
func acquireUnderlayGroupWithVersion(group loadGroup, index int, rule *VersionRule, accounting *importAccounting, versionState *sourceVersionState) (*composedDocument, []SourceInfo, map[string]ResourceContext, []Diagnostic) {
initialBytes, initialNodes := accounting.bytes, accounting.nodes
visited := make(map[resourceIdentity]importRoute)
hostToken := &namespaceToken{}
@@ -157,7 +161,7 @@ func acquireUnderlayGroup(group loadGroup, index int, rule *VersionRule, account
var contexts map[string]ResourceContext
for _, contribution := range group.inputs {
options := Options[struct{}]{Imports: contribution.imports, Filesystem: contribution.filesystem, Definition: Definition[struct{}]{Version: rule}}
acquired, diagnostics := assembleBaseInputTracked(contribution.source, options, contribution.destination, accounting, visited, hostToken)
acquired, diagnostics := assembleBaseInputTrackedForRole(contribution.source, options, contribution.destination, accounting, visited, hostToken, versionState, sourceVersionContribution)
for _, source := range acquired.sources {
source.Layer = LayerUnderlay
source.UnderlayIndex = index

220
internal/engine/source_version_test.go
View File

@@ -2,6 +2,7 @@ package engine
import (
"errors"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
@@ -754,6 +755,225 @@ configkit:
})
}
func TestSourceVersionUnderlaysCheckDistinctMountedDeclarations(t *testing.T) {
t.Parallel()
var checked []string
definition := Definition[map[string]any]{Version: &VersionRule{
Path: Keys("meta", "version"),
Check: func(version Version, validation *Validation) error {
checked = append(checked, version.Text)
if version.Text == "peer-a" || version.Text == "peer-c" {
validation.Warning(Path{}, "accepted with advisory")
}
return nil
},
}}
main := Input{
Source: Bytes("main", []byte("meta: {version: main-v}\nservice: primary\n")),
Destination: Keys("service"),
}
first := Input{
Source: Bytes("peer-a", []byte("meta:\n version: peer-a\nvalues: {first: one}\n")),
Destination: Keys("defaults", "first"),
}
second := Input{
Source: Bytes("peer-b", []byte("meta:\n version: peer-b\nvalues: {second: two}\n")),
Destination: Keys("defaults", "second"),
}
third := Input{
Source: Bytes("peer-c", []byte("meta:\n version: peer-c\nvalues: {third: three}\n")),
Destination: Keys("defaults", "third"),
}
loaded, err := LoadLayered(main, testSecretBinding(), LayeredOptions[map[string]any]{
Definition: definition, SourceVersionMode: sourceVersionModeRequired,
Underlays: []Underlay{{Inputs: []Input{first, second}}, {Inputs: []Input{third}}},
})
if err != nil {
t.Fatalf("LoadLayered: %v", err)
}
if len(checked) != 4 {
t.Fatalf("checked declarations = %#v, want main and all underlays", checked)
}
for _, want := range []string{"main-v", "peer-a", "peer-b", "peer-c"} {
found := false
for _, got := range checked {
found = found || got == want
}
if !found {
t.Errorf("checked declarations %#v do not include %q", checked, want)
}
}
if len(loaded.Inspection.Diagnostics()) != 2 || loaded.Inspection.Diagnostics()[0].Code != CodeVersionWarning || loaded.Inspection.Diagnostics()[1].Code != CodeVersionWarning {
t.Fatalf("accepted underlay warning was not retained: %#v", loaded.Inspection.Diagnostics())
}
sources := loaded.Inspection.Sources()
if len(sources) != 4 || sources[1].Layer != LayerUnderlay || sources[1].UnderlayIndex != 1 || sources[2].Layer != LayerUnderlay || sources[2].UnderlayIndex != 1 || sources[3].Layer != LayerUnderlay || sources[3].UnderlayIndex != 2 {
t.Fatalf("underlay source records = %#v", sources)
}
defaults, ok := loaded.Config["defaults"].(map[string]any)
if !ok {
t.Fatalf("defaults = %#v", loaded.Config["defaults"])
}
firstValue, ok := defaults["first"].(map[string]any)
if !ok || firstValue["values"] == nil {
t.Fatalf("first mounted contribution = %#v", defaults["first"])
}
if meta, ok := firstValue["meta"].(map[string]any); !ok || len(meta) != 0 {
t.Fatalf("consumed nested declaration did not preserve its empty ancestor: %#v", firstValue["meta"])
}
}
func TestSourceVersionUnderlaysRejectDeclarationsHiddenByMain(t *testing.T) {
t.Parallel()
var checked []string
loaded, err := LoadLayered(Input{Source: Bytes("main", []byte("meta: {version: main-v}\nservice: primary\n"))}, testSecretBinding(), LayeredOptions[map[string]any]{
Definition: Definition[map[string]any]{Version: &VersionRule{
Path: Keys("meta", "version"),
Check: func(version Version, _ *Validation) error {
checked = append(checked, version.Text)
if version.Text == "rejected-peer" {
return errors.New("peer declaration is not accepted")
}
return nil
},
}},
SourceVersionMode: sourceVersionModeIfPresent,
Underlays: []Underlay{{Inputs: []Input{{Source: Bytes("underlay", []byte("meta: {version: rejected-peer}\nservice: shadowed\n"))}}}},
})
if err == nil || loaded != nil {
t.Fatalf("LoadLayered = %#v, %v, want hidden underlay rejection", loaded, err)
}
loadErr, ok := err.(*LoadError)
if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeVersionHookFailed {
t.Fatalf("underlay rejection diagnostics = %#v", err)
}
if len(checked) != 2 || checked[0] != "main-v" || checked[1] != "rejected-peer" {
t.Fatalf("checked declarations = %#v", checked)
}
}
func TestSourceVersionUnderlaysRequireActualHeadersButSkipAbsentSources(t *testing.T) {
t.Parallel()
newOptions := func(underlays []Underlay) LayeredOptions[map[string]any] {
return LayeredOptions[map[string]any]{
Definition: Definition[map[string]any]{Version: &VersionRule{
Path: Keys("meta", "version"),
Check: func(Version, *Validation) error { return nil },
}},
SourceVersionMode: sourceVersionModeRequired,
Underlays: underlays,
}
}
main := Input{Source: Bytes("main", []byte("meta: {version: main-v}\n"))}
t.Run("required absence", func(t *testing.T) {
loaded, err := LoadLayered(main, testSecretBinding(), newOptions([]Underlay{{Inputs: []Input{{Source: Bytes("missing", []byte("value: present\n"))}}}}))
if err == nil || loaded != nil {
t.Fatalf("LoadLayered = %#v, %v, want missing underlay declaration", loaded, err)
}
loadErr, ok := err.(*LoadError)
if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeVersionError {
t.Fatalf("missing underlay diagnostics = %#v", err)
}
})
t.Run("empty group and NoSource", func(t *testing.T) {
loaded, err := LoadLayered(main, testSecretBinding(), newOptions([]Underlay{{}, {Inputs: []Input{{Source: NoSource()}}}}))
if err != nil {
t.Fatalf("LoadLayered with absent contributions: %v", err)
}
if sources := loaded.Inspection.Sources(); len(sources) != 1 {
t.Fatalf("empty groups and NoSource created source records: %#v", sources)
}
})
}
func TestSourceVersionUnderlayImportsKeepLocalHeadersAndMountedDestinations(t *testing.T) {
t.Parallel()
directory := t.TempDir()
underlayPath := writeImportFile(t, directory, "underlay.yml", `meta: {version: peer-root}
configkit:
imports:
- path: nested.yml
destination: [nested]
root: retained
`)
writeImportFile(t, directory, "nested.yml", "meta: {version: peer-nested}\nname: nested\n")
var checked []string
loaded, err := LoadLayered(Input{
Source: Bytes("main", []byte("meta: {version: main-v}\n")), Destination: Keys("service"),
}, testSecretBinding(), LayeredOptions[map[string]any]{
Definition: Definition[map[string]any]{Version: &VersionRule{
Path: Keys("meta", "version"),
Check: func(version Version, _ *Validation) error {
checked = append(checked, version.Text)
return nil
},
}},
SourceVersionMode: sourceVersionModeRequired,
Underlays: []Underlay{{Inputs: []Input{{
Source: File(underlayPath), Imports: true, Destination: Keys("defaults"),
}}}},
})
if err != nil {
t.Fatalf("LoadLayered: %v", err)
}
if len(checked) != 3 {
t.Fatalf("checked source declarations = %#v", checked)
}
for _, want := range []string{"main-v", "peer-root", "peer-nested"} {
found := false
for _, got := range checked {
found = found || got == want
}
if !found {
t.Errorf("checked source declarations %#v do not include %q", checked, want)
}
}
sources := loaded.Inspection.Sources()
if len(sources) != 3 || sources[1].Path != filepath.Clean(underlayPath) || sources[2].Destination.String() != "/defaults/nested" {
t.Fatalf("underlay and nested import records = %#v", sources)
}
if sources[1].Layer != LayerUnderlay || sources[1].UnderlayIndex != 1 || sources[2].Layer != LayerUnderlay || sources[2].UnderlayIndex != 1 {
t.Fatalf("underlay and nested import tiers = %#v", sources)
}
defaults, ok := loaded.Config["defaults"].(map[string]any)
if !ok || defaults["root"] != "retained" {
t.Fatalf("mounted underlay root = %#v", loaded.Config["defaults"])
}
nested, ok := defaults["nested"].(map[string]any)
if !ok || nested["name"] != "nested" {
t.Fatalf("mounted nested import = %#v", defaults["nested"])
}
}
func TestSourceVersionUnderlaysProtectRemainingMountedPayload(t *testing.T) {
t.Parallel()
loaded, err := LoadLayered(Input{
Source: Bytes("main", []byte("meta: {version: main-v}\n")), Destination: Keys("service"),
}, testSecretBinding(), LayeredOptions[map[string]any]{
Definition: Definition[map[string]any]{Version: &VersionRule{
Path: Keys("meta", "version"), Check: func(Version, *Validation) error { return nil },
}},
SourceVersionMode: sourceVersionModeRequired,
Underlays: []Underlay{{Inputs: []Input{{Source: Bytes("underlay", []byte(
"meta: {version: accepted}\nservice: {meta: {version: payload}}\n",
))}}}},
})
if err == nil || loaded != nil {
t.Fatalf("LoadLayered = %#v, %v, want protected remaining payload rejection", loaded, err)
}
loadErr, ok := err.(*LoadError)
if !ok || len(loadErr.Diagnostics()) != 1 || loadErr.Diagnostics()[0].Code != CodeUnderlayVersion {
t.Fatalf("remaining payload diagnostics = %#v", err)
}
}
func TestSourceVersionImportAccountingIncludesConsumedHeaderNodes(t *testing.T) {
t.Parallel()