Add recursive directory search with depth limits

internal/engine/directory_selection_test.go

@@ -4,6 +4,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"testing/fstest"
 )
@@ -38,7 +39,7 @@ func TestSelectShallowDirectoriesUnionsRootsAndPatterns(t *testing.T) {
 		t.Fatal(diagnostics)
 	}
 	accounting := &importAccounting{}
-	selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting)
+	selection, diagnostics := selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting)
 	if len(diagnostics) != 0 {
 		t.Fatalf("selection diagnostics = %#v", diagnostics)
 	}
@@ -85,7 +86,7 @@ func TestSelectShallowDirectoriesIncludeUnrestrictedHiddenAndExtensionlessFiles(
 	if len(diagnostics) != 0 {
 		t.Fatal(diagnostics)
 	}
-	selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
+	selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
 	if len(diagnostics) != 0 {
 		t.Fatalf("selection diagnostics = %#v", diagnostics)
 	}
@@ -144,7 +145,7 @@ func TestSelectShallowDirectoriesApplyRequirednessToFinalUnion(t *testing.T) {
 			if len(diagnostics) != 0 {
 				t.Fatal(diagnostics)
 			}
-			selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
+			selection, diagnostics := selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
 			if test.wantError {
 				if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing {
 					t.Fatalf("diagnostics = %#v, want required empty-union error", diagnostics)
@@ -176,7 +177,7 @@ func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t
 	if len(diagnostics) != 0 {
 		t.Fatal(diagnostics)
 	}
-	_, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
+	_, diagnostics = selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
 	if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
 		t.Fatalf("invalid-root diagnostics = %#v", diagnostics)
 	}
@@ -191,7 +192,7 @@ func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t
 	if len(diagnostics) != 0 {
 		t.Fatal(diagnostics)
 	}
-	_, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
+	_, diagnostics = selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
 	if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
 		t.Fatalf("inaccessible-root diagnostics = %#v", diagnostics)
 	}
@@ -221,8 +222,150 @@ func TestSelectShallowDirectoryFiltersSymlinkAliasSpelling(t *testing.T) {
 	if len(diagnostics) != 0 {
 		t.Fatal(diagnostics)
 	}
-	selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
+	selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
 	if len(diagnostics) != 0 || len(selection.candidates) != 1 || filepath.Base(selection.candidates[0].path) != "welcome.yaml" {
 		t.Fatalf("alias filtering selection, diagnostics = %#v, %#v", selection, diagnostics)
 	}
 }
+
+func TestSelectDirectoriesApplyRecursiveDepthToFileLevels(t *testing.T) {
+	t.Parallel()
+
+	root := t.TempDir()
+	for name := range map[string]struct{}{
+		"direct.yaml":            {},
+		"child/child.yaml":       {},
+		"child/grand/grand.yaml": {},
+	} {
+		filename := filepath.Join(root, filepath.FromSlash(name))
+		if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filename, []byte("value: true\n"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	access, err := openHostAccess(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer access.close()
+
+	for _, test := range []struct {
+		name      string
+		yaml      string
+		defaults  importDefaults
+		wantPaths []string
+	}{
+		{
+			name:      "depth one includes direct files",
+			yaml:      "    - dir: .\n      recursive: true\n      depth: 1\n      pattern: '*.yaml'\n",
+			wantPaths: []string{"direct.yaml"},
+		},
+		{
+			name:      "depth two includes one descendant level",
+			yaml:      "    - dir: .\n      recursive: true\n      depth: 2\n      pattern: '*.yaml'\n",
+			wantPaths: []string{"child.yaml", "direct.yaml"},
+		},
+		{
+			name:      "zero depth is unlimited",
+			yaml:      "    - dir: .\n      recursive: true\n      depth: 0\n      pattern: '*.yaml'\n",
+			wantPaths: []string{"child.yaml", "grand.yaml", "direct.yaml"},
+		},
+		{
+			name:      "recursion stays disabled by default",
+			yaml:      "    - dir: .\n      pattern: '*.yaml'\n",
+			wantPaths: []string{"direct.yaml"},
+		},
+		{
+			name:      "explicit false overrides application recursion",
+			yaml:      "    - dir: .\n      recursive: false\n      pattern: '*.yaml'\n",
+			defaults:  importDefaults{recursive: boolPointer(true)},
+			wantPaths: []string{"direct.yaml"},
+		},
+		{
+			name:      "application default enables recursion for depth",
+			yaml:      "    - dir: .\n      depth: 2\n      pattern: '*.yaml'\n",
+			defaults:  importDefaults{recursive: boolPointer(true)},
+			wantPaths: []string{"child.yaml", "direct.yaml"},
+		},
+		{
+			name:      "large depth remains a valid unlimited traversal bound",
+			yaml:      "    - dir: .\n      recursive: true\n      depth: 999999999999999999999999999999\n      pattern: '*.yaml'\n",
+			wantPaths: []string{"child.yaml", "grand.yaml", "direct.yaml"},
+		},
+		{
+			name:      "filename filter does not prune directories",
+			yaml:      "    - dir: .\n      recursive: true\n      depth: 3\n      pattern: grand.yaml\n",
+			wantPaths: []string{"grand.yaml"},
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			specs, diagnostics := parseImportSpecsYAML(t, test.yaml)
+			if len(diagnostics) != 0 {
+				t.Fatal(diagnostics)
+			}
+			policy, diagnostics := resolveImportPolicy(specs[0], test.defaults)
+			if len(diagnostics) != 0 {
+				t.Fatal(diagnostics)
+			}
+			selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
+			if len(diagnostics) != 0 {
+				t.Fatalf("selection diagnostics = %#v", diagnostics)
+			}
+			var got []string
+			for _, candidate := range selection.candidates {
+				got = append(got, filepath.Base(candidate.path))
+			}
+			if strings.Join(got, ",") != strings.Join(test.wantPaths, ",") {
+				t.Fatalf("selected basenames = %#v, want %#v", got, test.wantPaths)
+			}
+		})
+	}
+}
+
+func TestSelectDirectoriesRetainsEmptySearchAndDirectoryVisitLimits(t *testing.T) {
+	t.Parallel()
+
+	root := t.TempDir()
+	for _, name := range []string{"direct.yaml", "child/child.yaml", "child/grand/grand.yaml"} {
+		filename := filepath.Join(root, filepath.FromSlash(name))
+		if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filename, []byte("value: true\n"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	access, err := openHostAccess(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer access.close()
+
+	specs, diagnostics := parseImportSpecsYAML(t, "    - dir: .\n      recursive: true\n      pattern: []\n")
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	accounting := &importAccounting{}
+	selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), accounting)
+	if len(diagnostics) != 0 || selection.outcome != importSelectionNone || len(selection.candidates) != 0 {
+		t.Fatalf("empty recursive selection, diagnostics = %#v, %#v", selection, diagnostics)
+	}
+	if accounting.entries != 5 {
+		t.Fatalf("empty search visited %d entries, want all 5 within the recursive search", accounting.entries)
+	}
+
+	limited := defaultImportLimits()
+	limited.entries = 2
+	_, diagnostics = selectDirectoryImportCandidates(access, root, specs[0], policy, limited, &importAccounting{})
+	if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded {
+		t.Fatalf("directory limit diagnostics = %#v", diagnostics)
+	}
+}
+
+func boolPointer(value bool) *bool { return &value }

internal/engine/directory_walk.go

@@ -0,0 +1,50 @@
+package engine
+
+import (
+	"errors"
+	"io/fs"
+)
+
+var errImportDirectoryEntryLimit = errors.New("import directory-entry visit limit reached")
+
+// visitBoundedImportDirectory shares entry accounting across directory and glob searches.
+func visitBoundedImportDirectory(access *filesystemAccess, directory string, limit int, accounting *importAccounting, visit func(fs.DirEntry) error) error {
+	if accounting == nil {
+		accounting = &importAccounting{}
+	}
+	var visitError error
+	err := access.readDirectory(directory, func(entry fs.DirEntry) bool {
+		accounting.entries++
+		if accounting.entries > limit {
+			visitError = errImportDirectoryEntryLimit
+			return false
+		}
+		if err := visit(entry); err != nil {
+			visitError = err
+			return false
+		}
+		return true
+	})
+	if visitError != nil {
+		return visitError
+	}
+	return err
+}
+
+func importDirectoryEntryIsDirectory(entry fs.DirEntry) (bool, error) {
+	mode := entry.Type()
+	if mode&fs.ModeSymlink != 0 {
+		return false, nil
+	}
+	if mode.IsDir() {
+		return true, nil
+	}
+	if mode != 0 {
+		return false, nil
+	}
+	info, err := entry.Info()
+	if err != nil {
+		return false, err
+	}
+	return info.IsDir(), nil
+}

internal/engine/glob.go

@@ -10,6 +10,8 @@ import (
 
 const globReadBatch = 128
 
+var errInvalidImportGlobPattern = errors.New("invalid import glob filename pattern")
+
 type globCandidate struct {
 	absolute string
 	relative string
@@ -99,27 +101,21 @@ func (s *importTraversal) readGlobMatches(directory, pattern string, location Lo
 		return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory could not be addressed inside the configuration root", &location)}
 	}
 	var matches []string
-	var problem *Diagnostic
+	err = visitBoundedImportDirectory(s.access, canonical, s.limits.entries, s.accounting, func(entry fs.DirEntry) error {
-	err = s.access.readDirectory(canonical, func(entry fs.DirEntry) bool {
-		s.accounting.entries++
-		if s.accounting.entries > s.limits.entries {
-			value := importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location)
-			problem = &value
-			return false
-		}
 		matched, matchErr := path.Match(pattern, entry.Name())
 		if matchErr != nil {
-			value := importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location)
+			return errInvalidImportGlobPattern
-			problem = &value
-			return false
 		}
 		if matched {
 			matches = append(matches, entry.Name())
 		}
-		return true
+		return nil
 	})
-	if problem != nil {
+	if errors.Is(err, errImportDirectoryEntryLimit) {
-		return nil, []Diagnostic{*problem}
+		return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location)}
+	}
+	if errors.Is(err, errInvalidImportGlobPattern) {
+		return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location)}
 	}
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, nil

internal/engine/import_selection.go

@@ -72,7 +72,7 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec
 	return selection, nil
 }
 
-func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) {
+func selectDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) {
 	roots := make([]selectedImportCandidate, 0, len(spec.paths))
 	seenRoots := make(map[string]struct{}, len(spec.paths))
 	for index, relative := range spec.paths {
@@ -106,35 +106,53 @@ func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath s
 		if !isDirectory {
 			return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root is not a directory")}
 		}
-
+		entries, missing, diagnostic := importDirectoryEntries(access, root.accessPath, root.site, limits, accounting)
-		entries := make([]fs.DirEntry, 0)
+		if diagnostic != nil {
-		var problem *Diagnostic
+			return importSelection{}, []Diagnostic{*diagnostic}
-		err := access.readDirectory(root.accessPath, func(entry fs.DirEntry) bool {
-			accounting.entries++
-			if accounting.entries > limits.entries {
-				problem = importSelectionDiagnostic(root.site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit")
-				return false
 		}
-			entries = append(entries, entry)
+		if missing {
-			return true
-		})
-		if problem != nil {
-			return importSelection{}, []Diagnostic{*problem}
-		}
-		if errors.Is(err, fs.ErrNotExist) {
 			continue
 		}
-		if err != nil {
+		stack := []importDirectoryFrame{{directory: root, depth: 0, entries: entries}}
-			return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root could not be read")}
+		for len(stack) != 0 {
-		}
+			frameIndex := len(stack) - 1
-		sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() })
+			frame := &stack[frameIndex]
-		for _, entry := range entries {
+			if frame.next >= len(frame.entries) {
-			if entry.IsDir() || !matchesImportFilename(entry.Name(), policy.pattern) {
+				stack = stack[:frameIndex]
 				continue
 			}
-			accessPath := access.join(root.accessPath, entry.Name())
+			entry := frame.entries[frame.next]
+			frame.next++
+			accessPath := access.join(frame.directory.accessPath, entry.Name())
 			candidate := selectedImportCandidate{
-				path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: root.site,
+				path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: frame.directory.site,
+			}
+			isDirectory, entryErr := importDirectoryEntryIsDirectory(entry)
+			if entryErr != nil {
+				return importSelection{}, []Diagnostic{*importSelectionDiagnostic(frame.directory.site, CodeImportInvalid, "directory entry could not be inspected")}
+			}
+			if isDirectory {
+				if !directoryDepthAllows(policy, frame.depth) {
+					continue
+				}
+				childMissing, childIsDirectory, childDiagnostic := validateNamedImportTarget(access, candidate, false, true)
+				if childDiagnostic != nil {
+					return importSelection{}, []Diagnostic{*childDiagnostic}
+				}
+				if childMissing || !childIsDirectory {
+					continue
+				}
+				childEntries, vanished, childDiagnostic := importDirectoryEntries(access, candidate.accessPath, candidate.site, limits, accounting)
+				if childDiagnostic != nil {
+					return importSelection{}, []Diagnostic{*childDiagnostic}
+				}
+				if !vanished {
+					stack = append(stack, importDirectoryFrame{directory: candidate, depth: frame.depth + 1, entries: childEntries})
+				}
+				continue
+			}
+			if !matchesImportFilename(entry.Name(), policy.pattern) {
+				continue
 			}
 			if _, exists := seenCandidates[candidate.path]; exists {
 				continue
@@ -165,6 +183,40 @@ func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath s
 	return selection, nil
 }
 
+type importDirectoryFrame struct {
+	directory selectedImportCandidate
+	depth     int
+	entries   []fs.DirEntry
+	next      int
+}
+
+func importDirectoryEntries(access *filesystemAccess, directory string, site importFieldSite, limits importLimits, accounting *importAccounting) ([]fs.DirEntry, bool, *Diagnostic) {
+	entries := make([]fs.DirEntry, 0)
+	err := visitBoundedImportDirectory(access, directory, limits.entries, accounting, func(entry fs.DirEntry) error {
+		entries = append(entries, entry)
+		return nil
+	})
+	if errors.Is(err, errImportDirectoryEntryLimit) {
+		return nil, false, importSelectionDiagnostic(site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit")
+	}
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, true, nil
+	}
+	if err != nil {
+		return nil, false, importSelectionDiagnostic(site, CodeImportInvalid, "directory search root could not be read")
+	}
+	sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() })
+	return entries, false, nil
+}
+
+func directoryDepthAllows(policy resolvedImportPolicy, currentDepth int) bool {
+	if !policy.recursive {
+		return false
+	}
+	childDepth := currentDepth + 1
+	return !policy.depthSet || policy.depth == 0 || childDepth < policy.depth
+}
+
 func normalizedImportCandidatePath(access *filesystemAccess, accessPath string) string {
 	if access.native != nil {
 		return accessPath