eric/ConfigKit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add recursive directory search with depth limits

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-19 20:18:39 +00:00
parent 032679bf34
commit 77d97296ea
4 changed files with 285 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
internal/engine/directory_selection_test.go
View File

@@ -4,6 +4,7 @@ import (
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
)
@@ -38,7 +39,7 @@ func TestSelectShallowDirectoriesUnionsRootsAndPatterns(t *testing.T) {
t.Fatal(diagnostics)
}
accounting := &importAccounting{}
selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting)
selection, diagnostics := selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting)
if len(diagnostics) != 0 {
t.Fatalf("selection diagnostics = %#v", diagnostics)
}
@@ -85,7 +86,7 @@ func TestSelectShallowDirectoriesIncludeUnrestrictedHiddenAndExtensionlessFiles(
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 0 {
t.Fatalf("selection diagnostics = %#v", diagnostics)
}
@@ -144,7 +145,7 @@ func TestSelectShallowDirectoriesApplyRequirednessToFinalUnion(t *testing.T) {
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
selection, diagnostics := selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
if test.wantError {
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing {
t.Fatalf("diagnostics = %#v, want required empty-union error", diagnostics)
@@ -176,7 +177,7 @@ func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
_, diagnostics = selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
t.Fatalf("invalid-root diagnostics = %#v", diagnostics)
}
@@ -191,7 +192,7 @@ func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
_, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
_, diagnostics = selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
t.Fatalf("inaccessible-root diagnostics = %#v", diagnostics)
}
@@ -221,8 +222,150 @@ func TestSelectShallowDirectoryFiltersSymlinkAliasSpelling(t *testing.T) {
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 0 || len(selection.candidates) != 1 || filepath.Base(selection.candidates[0].path) != "welcome.yaml" {
t.Fatalf("alias filtering selection, diagnostics = %#v, %#v", selection, diagnostics)
}
}
func TestSelectDirectoriesApplyRecursiveDepthToFileLevels(t *testing.T) {
t.Parallel()
root := t.TempDir()
for name := range map[string]struct{}{
"direct.yaml": {},
"child/child.yaml": {},
"child/grand/grand.yaml": {},
} {
filename := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("value: true\n"), 0o600); err != nil {
t.Fatal(err)
}
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
for _, test := range []struct {
name string
yaml string
defaults importDefaults
wantPaths []string
}{
{
name: "depth one includes direct files",
yaml: " - dir: .\n recursive: true\n depth: 1\n pattern: '*.yaml'\n",
wantPaths: []string{"direct.yaml"},
},
{
name: "depth two includes one descendant level",
yaml: " - dir: .\n recursive: true\n depth: 2\n pattern: '*.yaml'\n",
wantPaths: []string{"child.yaml", "direct.yaml"},
},
{
name: "zero depth is unlimited",
yaml: " - dir: .\n recursive: true\n depth: 0\n pattern: '*.yaml'\n",
wantPaths: []string{"child.yaml", "grand.yaml", "direct.yaml"},
},
{
name: "recursion stays disabled by default",
yaml: " - dir: .\n pattern: '*.yaml'\n",
wantPaths: []string{"direct.yaml"},
},
{
name: "explicit false overrides application recursion",
yaml: " - dir: .\n recursive: false\n pattern: '*.yaml'\n",
defaults: importDefaults{recursive: boolPointer(true)},
wantPaths: []string{"direct.yaml"},
},
{
name: "application default enables recursion for depth",
yaml: " - dir: .\n depth: 2\n pattern: '*.yaml'\n",
defaults: importDefaults{recursive: boolPointer(true)},
wantPaths: []string{"child.yaml", "direct.yaml"},
},
{
name: "large depth remains a valid unlimited traversal bound",
yaml: " - dir: .\n recursive: true\n depth: 999999999999999999999999999999\n pattern: '*.yaml'\n",
wantPaths: []string{"child.yaml", "grand.yaml", "direct.yaml"},
},
{
name: "filename filter does not prune directories",
yaml: " - dir: .\n recursive: true\n depth: 3\n pattern: grand.yaml\n",
wantPaths: []string{"grand.yaml"},
},
} {
t.Run(test.name, func(t *testing.T) {
specs, diagnostics := parseImportSpecsYAML(t, test.yaml)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], test.defaults)
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
if len(diagnostics) != 0 {
t.Fatalf("selection diagnostics = %#v", diagnostics)
}
var got []string
for _, candidate := range selection.candidates {
got = append(got, filepath.Base(candidate.path))
}
if strings.Join(got, ",") != strings.Join(test.wantPaths, ",") {
t.Fatalf("selected basenames = %#v, want %#v", got, test.wantPaths)
}
})
}
}
func TestSelectDirectoriesRetainsEmptySearchAndDirectoryVisitLimits(t *testing.T) {
t.Parallel()
root := t.TempDir()
for _, name := range []string{"direct.yaml", "child/child.yaml", "child/grand/grand.yaml"} {
filename := filepath.Join(root, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("value: true\n"), 0o600); err != nil {
t.Fatal(err)
}
}
access, err := openHostAccess(root)
if err != nil {
t.Fatal(err)
}
defer access.close()
specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n recursive: true\n pattern: []\n")
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
accounting := &importAccounting{}
selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), accounting)
if len(diagnostics) != 0 || selection.outcome != importSelectionNone || len(selection.candidates) != 0 {
t.Fatalf("empty recursive selection, diagnostics = %#v, %#v", selection, diagnostics)
}
if accounting.entries != 5 {
t.Fatalf("empty search visited %d entries, want all 5 within the recursive search", accounting.entries)
}
limited := defaultImportLimits()
limited.entries = 2
_, diagnostics = selectDirectoryImportCandidates(access, root, specs[0], policy, limited, &importAccounting{})
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
t.Fatalf("directory limit diagnostics = %#v", diagnostics)
}
}
func boolPointer(value bool) *bool { return &value }

50
internal/engine/directory_walk.go Normal file
View File

@@ -0,0 +1,50 @@
package engine
import (
"errors"
"io/fs"
)
var errImportDirectoryEntryLimit = errors.New("import directory-entry visit limit reached")
// visitBoundedImportDirectory shares entry accounting across directory and glob searches.
func visitBoundedImportDirectory(access *filesystemAccess, directory string, limit int, accounting *importAccounting, visit func(fs.DirEntry) error) error {
if accounting == nil {
accounting = &importAccounting{}
}
var visitError error
err := access.readDirectory(directory, func(entry fs.DirEntry) bool {
accounting.entries++
if accounting.entries > limit {
visitError = errImportDirectoryEntryLimit
return false
}
if err := visit(entry); err != nil {
visitError = err
return false
}
return true
})
if visitError != nil {
return visitError
}
return err
}
func importDirectoryEntryIsDirectory(entry fs.DirEntry) (bool, error) {
mode := entry.Type()
if mode&fs.ModeSymlink != 0 {
return false, nil
}
if mode.IsDir() {
return true, nil
}
if mode != 0 {
return false, nil
}
info, err := entry.Info()
if err != nil {
return false, err
}
return info.IsDir(), nil
}

24
internal/engine/glob.go
View File

@@ -10,6 +10,8 @@ import (
const globReadBatch = 128
var errInvalidImportGlobPattern = errors.New("invalid import glob filename pattern")
type globCandidate struct {
absolute string
relative string
@@ -99,27 +101,21 @@ func (s *importTraversal) readGlobMatches(directory, pattern string, location Lo
return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory could not be addressed inside the configuration root", &location)}
}
var matches []string
var problem *Diagnostic
err = s.access.readDirectory(canonical, func(entry fs.DirEntry) bool {
s.accounting.entries++
if s.accounting.entries > s.limits.entries {
value := importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location)
problem = &value
return false
}
err = visitBoundedImportDirectory(s.access, canonical, s.limits.entries, s.accounting, func(entry fs.DirEntry) error {
matched, matchErr := path.Match(pattern, entry.Name())
if matchErr != nil {
value := importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location)
problem = &value
return false
return errInvalidImportGlobPattern
}
if matched {
matches = append(matches, entry.Name())
}
return true
return nil
})
if problem != nil {
return nil, []Diagnostic{*problem}
if errors.Is(err, errImportDirectoryEntryLimit) {
return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location)}
}
if errors.Is(err, errInvalidImportGlobPattern) {
return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location)}
}
if errors.Is(err, fs.ErrNotExist) {
return nil, nil

100
internal/engine/import_selection.go
View File

@@ -72,7 +72,7 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec
return selection, nil
}
func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) {
func selectDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) {
roots := make([]selectedImportCandidate, 0, len(spec.paths))
seenRoots := make(map[string]struct{}, len(spec.paths))
for index, relative := range spec.paths {
@@ -106,35 +106,53 @@ func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath s
if !isDirectory {
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root is not a directory")}
}
entries := make([]fs.DirEntry, 0)
var problem *Diagnostic
err := access.readDirectory(root.accessPath, func(entry fs.DirEntry) bool {
accounting.entries++
if accounting.entries > limits.entries {
problem = importSelectionDiagnostic(root.site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit")
return false
}
entries = append(entries, entry)
return true
})
if problem != nil {
return importSelection{}, []Diagnostic{*problem}
entries, missing, diagnostic := importDirectoryEntries(access, root.accessPath, root.site, limits, accounting)
if diagnostic != nil {
return importSelection{}, []Diagnostic{*diagnostic}
}
if errors.Is(err, fs.ErrNotExist) {
if missing {
continue
}
if err != nil {
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root could not be read")}
}
sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() })
for _, entry := range entries {
if entry.IsDir() || !matchesImportFilename(entry.Name(), policy.pattern) {
stack := []importDirectoryFrame{{directory: root, depth: 0, entries: entries}}
for len(stack) != 0 {
frameIndex := len(stack) - 1
frame := &stack[frameIndex]
if frame.next >= len(frame.entries) {
stack = stack[:frameIndex]
continue
}
accessPath := access.join(root.accessPath, entry.Name())
entry := frame.entries[frame.next]
frame.next++
accessPath := access.join(frame.directory.accessPath, entry.Name())
candidate := selectedImportCandidate{
path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: root.site,
path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: frame.directory.site,
}
isDirectory, entryErr := importDirectoryEntryIsDirectory(entry)
if entryErr != nil {
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(frame.directory.site, CodeImportInvalid, "directory entry could not be inspected")}
}
if isDirectory {
if !directoryDepthAllows(policy, frame.depth) {
continue
}
childMissing, childIsDirectory, childDiagnostic := validateNamedImportTarget(access, candidate, false, true)
if childDiagnostic != nil {
return importSelection{}, []Diagnostic{*childDiagnostic}
}
if childMissing || !childIsDirectory {
continue
}
childEntries, vanished, childDiagnostic := importDirectoryEntries(access, candidate.accessPath, candidate.site, limits, accounting)
if childDiagnostic != nil {
return importSelection{}, []Diagnostic{*childDiagnostic}
}
if !vanished {
stack = append(stack, importDirectoryFrame{directory: candidate, depth: frame.depth + 1, entries: childEntries})
}
continue
}
if !matchesImportFilename(entry.Name(), policy.pattern) {
continue
}
if _, exists := seenCandidates[candidate.path]; exists {
continue
@@ -165,6 +183,40 @@ func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath s
return selection, nil
}
type importDirectoryFrame struct {
directory selectedImportCandidate
depth int
entries []fs.DirEntry
next int
}
func importDirectoryEntries(access *filesystemAccess, directory string, site importFieldSite, limits importLimits, accounting *importAccounting) ([]fs.DirEntry, bool, *Diagnostic) {
entries := make([]fs.DirEntry, 0)
err := visitBoundedImportDirectory(access, directory, limits.entries, accounting, func(entry fs.DirEntry) error {
entries = append(entries, entry)
return nil
})
if errors.Is(err, errImportDirectoryEntryLimit) {
return nil, false, importSelectionDiagnostic(site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit")
}
if errors.Is(err, fs.ErrNotExist) {
return nil, true, nil
}
if err != nil {
return nil, false, importSelectionDiagnostic(site, CodeImportInvalid, "directory search root could not be read")
}
sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() })
return entries, false, nil
}
func directoryDepthAllows(policy resolvedImportPolicy, currentDepth int) bool {
if !policy.recursive {
return false
}
childDepth := currentDepth + 1
return !policy.depthSet || policy.depth == 0 || childDepth < policy.depth
}
func normalizedImportCandidatePath(access *filesystemAccess, accessPath string) string {
if access.native != nil {
return accessPath