diff --git a/internal/engine/directory_selection_test.go b/internal/engine/directory_selection_test.go index 132bfa1..bb3c47c 100644 --- a/internal/engine/directory_selection_test.go +++ b/internal/engine/directory_selection_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "testing" "testing/fstest" ) @@ -38,7 +39,7 @@ func TestSelectShallowDirectoriesUnionsRootsAndPatterns(t *testing.T) { t.Fatal(diagnostics) } accounting := &importAccounting{} - selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting) + selection, diagnostics := selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting) if len(diagnostics) != 0 { t.Fatalf("selection diagnostics = %#v", diagnostics) } @@ -85,7 +86,7 @@ func TestSelectShallowDirectoriesIncludeUnrestrictedHiddenAndExtensionlessFiles( if len(diagnostics) != 0 { t.Fatal(diagnostics) } - selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{}) + selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{}) if len(diagnostics) != 0 { t.Fatalf("selection diagnostics = %#v", diagnostics) } @@ -144,7 +145,7 @@ func TestSelectShallowDirectoriesApplyRequirednessToFinalUnion(t *testing.T) { if len(diagnostics) != 0 { t.Fatal(diagnostics) } - selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) + selection, diagnostics := selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) if test.wantError { if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing { t.Fatalf("diagnostics = %#v, want required empty-union error", diagnostics) @@ -176,7 +177,7 @@ func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t if len(diagnostics) != 0 { t.Fatal(diagnostics) } - _, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) + _, diagnostics = selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid { t.Fatalf("invalid-root diagnostics = %#v", diagnostics) } @@ -191,7 +192,7 @@ func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t if len(diagnostics) != 0 { t.Fatal(diagnostics) } - _, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) + _, diagnostics = selectDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid { t.Fatalf("inaccessible-root diagnostics = %#v", diagnostics) } @@ -221,8 +222,150 @@ func TestSelectShallowDirectoryFiltersSymlinkAliasSpelling(t *testing.T) { if len(diagnostics) != 0 { t.Fatal(diagnostics) } - selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{}) + selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{}) if len(diagnostics) != 0 || len(selection.candidates) != 1 || filepath.Base(selection.candidates[0].path) != "welcome.yaml" { t.Fatalf("alias filtering selection, diagnostics = %#v, %#v", selection, diagnostics) } } + +func TestSelectDirectoriesApplyRecursiveDepthToFileLevels(t *testing.T) { + t.Parallel() + + root := t.TempDir() + for name := range map[string]struct{}{ + "direct.yaml": {}, + "child/child.yaml": {}, + "child/grand/grand.yaml": {}, + } { + filename := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte("value: true\n"), 0o600); err != nil { + t.Fatal(err) + } + } + access, err := openHostAccess(root) + if err != nil { + t.Fatal(err) + } + defer access.close() + + for _, test := range []struct { + name string + yaml string + defaults importDefaults + wantPaths []string + }{ + { + name: "depth one includes direct files", + yaml: " - dir: .\n recursive: true\n depth: 1\n pattern: '*.yaml'\n", + wantPaths: []string{"direct.yaml"}, + }, + { + name: "depth two includes one descendant level", + yaml: " - dir: .\n recursive: true\n depth: 2\n pattern: '*.yaml'\n", + wantPaths: []string{"child.yaml", "direct.yaml"}, + }, + { + name: "zero depth is unlimited", + yaml: " - dir: .\n recursive: true\n depth: 0\n pattern: '*.yaml'\n", + wantPaths: []string{"child.yaml", "grand.yaml", "direct.yaml"}, + }, + { + name: "recursion stays disabled by default", + yaml: " - dir: .\n pattern: '*.yaml'\n", + wantPaths: []string{"direct.yaml"}, + }, + { + name: "explicit false overrides application recursion", + yaml: " - dir: .\n recursive: false\n pattern: '*.yaml'\n", + defaults: importDefaults{recursive: boolPointer(true)}, + wantPaths: []string{"direct.yaml"}, + }, + { + name: "application default enables recursion for depth", + yaml: " - dir: .\n depth: 2\n pattern: '*.yaml'\n", + defaults: importDefaults{recursive: boolPointer(true)}, + wantPaths: []string{"child.yaml", "direct.yaml"}, + }, + { + name: "large depth remains a valid unlimited traversal bound", + yaml: " - dir: .\n recursive: true\n depth: 999999999999999999999999999999\n pattern: '*.yaml'\n", + wantPaths: []string{"child.yaml", "grand.yaml", "direct.yaml"}, + }, + { + name: "filename filter does not prune directories", + yaml: " - dir: .\n recursive: true\n depth: 3\n pattern: grand.yaml\n", + wantPaths: []string{"grand.yaml"}, + }, + } { + t.Run(test.name, func(t *testing.T) { + specs, diagnostics := parseImportSpecsYAML(t, test.yaml) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + policy, diagnostics := resolveImportPolicy(specs[0], test.defaults) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{}) + if len(diagnostics) != 0 { + t.Fatalf("selection diagnostics = %#v", diagnostics) + } + var got []string + for _, candidate := range selection.candidates { + got = append(got, filepath.Base(candidate.path)) + } + if strings.Join(got, ",") != strings.Join(test.wantPaths, ",") { + t.Fatalf("selected basenames = %#v, want %#v", got, test.wantPaths) + } + }) + } +} + +func TestSelectDirectoriesRetainsEmptySearchAndDirectoryVisitLimits(t *testing.T) { + t.Parallel() + + root := t.TempDir() + for _, name := range []string{"direct.yaml", "child/child.yaml", "child/grand/grand.yaml"} { + filename := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte("value: true\n"), 0o600); err != nil { + t.Fatal(err) + } + } + access, err := openHostAccess(root) + if err != nil { + t.Fatal(err) + } + defer access.close() + + specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n recursive: true\n pattern: []\n") + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + accounting := &importAccounting{} + selection, diagnostics := selectDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), accounting) + if len(diagnostics) != 0 || selection.outcome != importSelectionNone || len(selection.candidates) != 0 { + t.Fatalf("empty recursive selection, diagnostics = %#v, %#v", selection, diagnostics) + } + if accounting.entries != 5 { + t.Fatalf("empty search visited %d entries, want all 5 within the recursive search", accounting.entries) + } + + limited := defaultImportLimits() + limited.entries = 2 + _, diagnostics = selectDirectoryImportCandidates(access, root, specs[0], policy, limited, &importAccounting{}) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportLimitExceeded { + t.Fatalf("directory limit diagnostics = %#v", diagnostics) + } +} + +func boolPointer(value bool) *bool { return &value } diff --git a/internal/engine/directory_walk.go b/internal/engine/directory_walk.go new file mode 100644 index 0000000..97c566d --- /dev/null +++ b/internal/engine/directory_walk.go @@ -0,0 +1,50 @@ +package engine + +import ( + "errors" + "io/fs" +) + +var errImportDirectoryEntryLimit = errors.New("import directory-entry visit limit reached") + +// visitBoundedImportDirectory shares entry accounting across directory and glob searches. +func visitBoundedImportDirectory(access *filesystemAccess, directory string, limit int, accounting *importAccounting, visit func(fs.DirEntry) error) error { + if accounting == nil { + accounting = &importAccounting{} + } + var visitError error + err := access.readDirectory(directory, func(entry fs.DirEntry) bool { + accounting.entries++ + if accounting.entries > limit { + visitError = errImportDirectoryEntryLimit + return false + } + if err := visit(entry); err != nil { + visitError = err + return false + } + return true + }) + if visitError != nil { + return visitError + } + return err +} + +func importDirectoryEntryIsDirectory(entry fs.DirEntry) (bool, error) { + mode := entry.Type() + if mode&fs.ModeSymlink != 0 { + return false, nil + } + if mode.IsDir() { + return true, nil + } + if mode != 0 { + return false, nil + } + info, err := entry.Info() + if err != nil { + return false, err + } + return info.IsDir(), nil +} diff --git a/internal/engine/glob.go b/internal/engine/glob.go index 749b6ac..3216fbc 100644 --- a/internal/engine/glob.go +++ b/internal/engine/glob.go @@ -10,6 +10,8 @@ import ( const globReadBatch = 128 +var errInvalidImportGlobPattern = errors.New("invalid import glob filename pattern") + type globCandidate struct { absolute string relative string @@ -99,27 +101,21 @@ func (s *importTraversal) readGlobMatches(directory, pattern string, location Lo return nil, []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "glob directory could not be addressed inside the configuration root", &location)} } var matches []string - var problem *Diagnostic - err = s.access.readDirectory(canonical, func(entry fs.DirEntry) bool { - s.accounting.entries++ - if s.accounting.entries > s.limits.entries { - value := importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location) - problem = &value - return false - } + err = visitBoundedImportDirectory(s.access, canonical, s.limits.entries, s.accounting, func(entry fs.DirEntry) error { matched, matchErr := path.Match(pattern, entry.Name()) if matchErr != nil { - value := importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location) - problem = &value - return false + return errInvalidImportGlobPattern } if matched { matches = append(matches, entry.Name()) } - return true + return nil }) - if problem != nil { - return nil, []Diagnostic{*problem} + if errors.Is(err, errImportDirectoryEntryLimit) { + return nil, []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "glob traversal exceeds the directory-entry visit limit", &location)} + } + if errors.Is(err, errInvalidImportGlobPattern) { + return nil, []Diagnostic{importDiagnostic(CodeImportInvalid, "import glob pattern is invalid", &location)} } if errors.Is(err, fs.ErrNotExist) { return nil, nil diff --git a/internal/engine/import_selection.go b/internal/engine/import_selection.go index bfd904e..9e5ec31 100644 --- a/internal/engine/import_selection.go +++ b/internal/engine/import_selection.go @@ -72,7 +72,7 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec return selection, nil } -func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) { +func selectDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) { roots := make([]selectedImportCandidate, 0, len(spec.paths)) seenRoots := make(map[string]struct{}, len(spec.paths)) for index, relative := range spec.paths { @@ -106,35 +106,53 @@ func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath s if !isDirectory { return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root is not a directory")} } - - entries := make([]fs.DirEntry, 0) - var problem *Diagnostic - err := access.readDirectory(root.accessPath, func(entry fs.DirEntry) bool { - accounting.entries++ - if accounting.entries > limits.entries { - problem = importSelectionDiagnostic(root.site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit") - return false - } - entries = append(entries, entry) - return true - }) - if problem != nil { - return importSelection{}, []Diagnostic{*problem} + entries, missing, diagnostic := importDirectoryEntries(access, root.accessPath, root.site, limits, accounting) + if diagnostic != nil { + return importSelection{}, []Diagnostic{*diagnostic} } - if errors.Is(err, fs.ErrNotExist) { + if missing { continue } - if err != nil { - return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root could not be read")} - } - sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() }) - for _, entry := range entries { - if entry.IsDir() || !matchesImportFilename(entry.Name(), policy.pattern) { + stack := []importDirectoryFrame{{directory: root, depth: 0, entries: entries}} + for len(stack) != 0 { + frameIndex := len(stack) - 1 + frame := &stack[frameIndex] + if frame.next >= len(frame.entries) { + stack = stack[:frameIndex] continue } - accessPath := access.join(root.accessPath, entry.Name()) + entry := frame.entries[frame.next] + frame.next++ + accessPath := access.join(frame.directory.accessPath, entry.Name()) candidate := selectedImportCandidate{ - path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: root.site, + path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: frame.directory.site, + } + isDirectory, entryErr := importDirectoryEntryIsDirectory(entry) + if entryErr != nil { + return importSelection{}, []Diagnostic{*importSelectionDiagnostic(frame.directory.site, CodeImportInvalid, "directory entry could not be inspected")} + } + if isDirectory { + if !directoryDepthAllows(policy, frame.depth) { + continue + } + childMissing, childIsDirectory, childDiagnostic := validateNamedImportTarget(access, candidate, false, true) + if childDiagnostic != nil { + return importSelection{}, []Diagnostic{*childDiagnostic} + } + if childMissing || !childIsDirectory { + continue + } + childEntries, vanished, childDiagnostic := importDirectoryEntries(access, candidate.accessPath, candidate.site, limits, accounting) + if childDiagnostic != nil { + return importSelection{}, []Diagnostic{*childDiagnostic} + } + if !vanished { + stack = append(stack, importDirectoryFrame{directory: candidate, depth: frame.depth + 1, entries: childEntries}) + } + continue + } + if !matchesImportFilename(entry.Name(), policy.pattern) { + continue } if _, exists := seenCandidates[candidate.path]; exists { continue @@ -165,6 +183,40 @@ func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath s return selection, nil } +type importDirectoryFrame struct { + directory selectedImportCandidate + depth int + entries []fs.DirEntry + next int +} + +func importDirectoryEntries(access *filesystemAccess, directory string, site importFieldSite, limits importLimits, accounting *importAccounting) ([]fs.DirEntry, bool, *Diagnostic) { + entries := make([]fs.DirEntry, 0) + err := visitBoundedImportDirectory(access, directory, limits.entries, accounting, func(entry fs.DirEntry) error { + entries = append(entries, entry) + return nil + }) + if errors.Is(err, errImportDirectoryEntryLimit) { + return nil, false, importSelectionDiagnostic(site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit") + } + if errors.Is(err, fs.ErrNotExist) { + return nil, true, nil + } + if err != nil { + return nil, false, importSelectionDiagnostic(site, CodeImportInvalid, "directory search root could not be read") + } + sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() }) + return entries, false, nil +} + +func directoryDepthAllows(policy resolvedImportPolicy, currentDepth int) bool { + if !policy.recursive { + return false + } + childDepth := currentDepth + 1 + return !policy.depthSet || policy.depth == 0 || childDepth < policy.depth +} + func normalizedImportCandidatePath(access *filesystemAccess, accessPath string) string { if access.native != nil { return accessPath