diff --git a/internal/engine/directory_selection_test.go b/internal/engine/directory_selection_test.go new file mode 100644 index 0000000..132bfa1 --- /dev/null +++ b/internal/engine/directory_selection_test.go @@ -0,0 +1,228 @@ +package engine + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + "testing/fstest" +) + +func TestSelectShallowDirectoriesUnionsRootsAndPatterns(t *testing.T) { + t.Parallel() + + filesystem := fstest.MapFS{ + "one": &fstest.MapFile{Mode: fs.ModeDir | 0o700}, + "one/a.yaml": &fstest.MapFile{Data: []byte("a: 1\n")}, + "one/README": &fstest.MapFile{Data: []byte("readme\n")}, + "one/.hidden": &fstest.MapFile{Data: []byte("hidden\n")}, + "one/draft.draft.yaml": &fstest.MapFile{Data: []byte("draft\n")}, + "one/nested/deep.yaml": &fstest.MapFile{Data: []byte("deep: 1\n")}, + "two": &fstest.MapFile{Mode: fs.ModeDir | 0o700}, + "two/a.yaml": &fstest.MapFile{Data: []byte("a: 2\n")}, + "three": &fstest.MapFile{Mode: fs.ModeDir | 0o700}, + "three/extensionless": &fstest.MapFile{Data: []byte("plain\n")}, + } + access := newNativeAccess(filesystem, ".", &namespaceToken{}) + specs, diagnostics := parseImportSpecsYAML(t, ` + - dir: [one, one, missing, two] + pattern: ['*.yaml', 'a.*', 'README'] + exclude: '*.draft.yaml' + required: true +`) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + accounting := &importAccounting{} + selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting) + if len(diagnostics) != 0 { + t.Fatalf("selection diagnostics = %#v", diagnostics) + } + if selection.outcome != importSelectionHasCandidates || len(selection.candidates) != 3 { + t.Fatalf("selection = %#v, want three unioned matches", selection) + } + if selection.candidates[0].path != "one/README" || selection.candidates[1].path != "one/a.yaml" || selection.candidates[2].path != "two/a.yaml" { + t.Fatalf("selected paths = %#v", selection.candidates) + } + if accounting.entries != 6 { + t.Fatalf("entry visits = %d, want 6 entries across distinct roots", accounting.entries) + } +} + +func TestSelectShallowDirectoriesIncludeUnrestrictedHiddenAndExtensionlessFiles(t *testing.T) { + t.Parallel() + + root := t.TempDir() + for name, value := range map[string]string{ + "a.yaml": "a: 1\n", + ".private": "private\n", + "README": "readme\n", + "nested/deep": "deep\n", + } { + filename := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte(value), 0o600); err != nil { + t.Fatal(err) + } + } + access, err := openHostAccess(root) + if err != nil { + t.Fatal(err) + } + defer access.close() + + specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n") + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{}) + if len(diagnostics) != 0 { + t.Fatalf("selection diagnostics = %#v", diagnostics) + } + if len(selection.candidates) != 3 { + t.Fatalf("selected files = %#v, want yaml, hidden, and extensionless files", selection.candidates) + } + for _, candidate := range selection.candidates { + if filepath.Base(candidate.path) == "deep" { + t.Fatalf("shallow search descended into a child directory: %#v", selection.candidates) + } + } +} + +func TestSelectShallowDirectoriesApplyRequirednessToFinalUnion(t *testing.T) { + t.Parallel() + + filesystem := fstest.MapFS{ + "available": &fstest.MapFile{Mode: fs.ModeDir | 0o700}, + "available/match.yaml": &fstest.MapFile{Data: []byte("match: true\n")}, + "available/excluded.yaml": &fstest.MapFile{Data: []byte("excluded: true\n")}, + } + access := newNativeAccess(filesystem, ".", &namespaceToken{}) + for _, test := range []struct { + name string + yaml string + wantError bool + wantCount int + }{ + { + name: "missing root alongside a match", + yaml: " - dir: [missing, available]\n pattern: '*.yaml'\n exclude: excluded.yaml\n required: true\n", + wantCount: 1, + }, + { + name: "all matches excluded", + yaml: " - dir: available\n pattern: '*.yaml'\n exclude: '*.yaml'\n required: true\n", + wantError: true, + }, + { + name: "explicit empty patterns select nothing", + yaml: " - dir: available\n pattern: []\n", + wantCount: 0, + }, + { + name: "empty search roots", + yaml: " - dir: []\n required: true\n", + wantError: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + specs, diagnostics := parseImportSpecsYAML(t, test.yaml) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) + if test.wantError { + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing { + t.Fatalf("diagnostics = %#v, want required empty-union error", diagnostics) + } + return + } + if len(diagnostics) != 0 || len(selection.candidates) != test.wantCount { + t.Fatalf("selection, diagnostics = %#v, %#v", selection, diagnostics) + } + if test.wantCount == 1 && selection.candidates[0].path != "available/match.yaml" { + t.Fatalf("selected path = %q, want available/match.yaml", selection.candidates[0].path) + } + }) + } +} + +func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t *testing.T) { + t.Parallel() + + filesystem := fstest.MapFS{ + "file.yaml": &fstest.MapFile{Data: []byte("not a directory\n")}, + } + access := newNativeAccess(filesystem, ".", &namespaceToken{}) + specs, diagnostics := parseImportSpecsYAML(t, " - dir: file.yaml\n") + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + _, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid { + t.Fatalf("invalid-root diagnostics = %#v", diagnostics) + } + + denied := &statOnlyImportFilesystem{statErrors: map[string]error{"denied": fs.ErrPermission}} + access = newNativeAccess(denied, ".", &namespaceToken{}) + specs, diagnostics = parseImportSpecsYAML(t, " - dir: denied\n") + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + policy, diagnostics = resolveImportPolicy(specs[0], importDefaults{}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + _, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{}) + if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid { + t.Fatalf("inaccessible-root diagnostics = %#v", diagnostics) + } +} + +func TestSelectShallowDirectoryFiltersSymlinkAliasSpelling(t *testing.T) { + t.Parallel() + + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "welcome.yaml"), []byte("welcome: true\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink("welcome.yaml", filepath.Join(root, "welcome.draft.yaml")); err != nil { + t.Skipf("symlinks are unavailable: %v", err) + } + access, err := openHostAccess(root) + if err != nil { + t.Fatal(err) + } + defer access.close() + + specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n pattern: '*.yaml'\n exclude: '*.draft.yaml'\n") + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{}) + if len(diagnostics) != 0 { + t.Fatal(diagnostics) + } + selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{}) + if len(diagnostics) != 0 || len(selection.candidates) != 1 || filepath.Base(selection.candidates[0].path) != "welcome.yaml" { + t.Fatalf("alias filtering selection, diagnostics = %#v, %#v", selection, diagnostics) + } +} diff --git a/internal/engine/import_selection.go b/internal/engine/import_selection.go index 1dc44f6..bfd904e 100644 --- a/internal/engine/import_selection.go +++ b/internal/engine/import_selection.go @@ -41,8 +41,8 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec } seen[candidatePath] = struct{}{} var site importFieldSite - if index < len(spec.pathSites) { - site = spec.pathSites[index] + if index < len(spec.selectorSites) { + site = spec.selectorSites[index] } else { site = spec.fieldSites["path"] } @@ -53,7 +53,7 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec }) candidates := make([]selectedImportCandidate, 0, len(normalized)) for _, candidate := range normalized { - missing, diagnostic := validateNamedImportTarget(access, candidate, policy.required) + missing, _, diagnostic := validateNamedImportTarget(access, candidate, policy.required, false) if diagnostic != nil { return importSelection{}, []Diagnostic{*diagnostic} } @@ -72,43 +72,159 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec return selection, nil } -func validateNamedImportTarget(access *filesystemAccess, candidate selectedImportCandidate, required bool) (bool, *Diagnostic) { +func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) { + roots := make([]selectedImportCandidate, 0, len(spec.paths)) + seenRoots := make(map[string]struct{}, len(spec.paths)) + for index, relative := range spec.paths { + accessPath := access.join(basePath, relative) + candidatePath := normalizedImportCandidatePath(access, accessPath) + if _, exists := seenRoots[candidatePath]; exists { + continue + } + seenRoots[candidatePath] = struct{}{} + site := spec.fieldSites["dir"] + if index < len(spec.selectorSites) { + site = spec.selectorSites[index] + } + roots = append(roots, selectedImportCandidate{path: candidatePath, accessPath: accessPath, site: site}) + } + sort.Slice(roots, func(left, right int) bool { return roots[left].path < roots[right].path }) + + if accounting == nil { + accounting = &importAccounting{} + } + discovered := make([]selectedImportCandidate, 0) + seenCandidates := make(map[string]struct{}) + for _, root := range roots { + missing, isDirectory, diagnostic := validateNamedImportTarget(access, root, false, true) + if diagnostic != nil { + return importSelection{}, []Diagnostic{*diagnostic} + } + if missing { + continue + } + if !isDirectory { + return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root is not a directory")} + } + + entries := make([]fs.DirEntry, 0) + var problem *Diagnostic + err := access.readDirectory(root.accessPath, func(entry fs.DirEntry) bool { + accounting.entries++ + if accounting.entries > limits.entries { + problem = importSelectionDiagnostic(root.site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit") + return false + } + entries = append(entries, entry) + return true + }) + if problem != nil { + return importSelection{}, []Diagnostic{*problem} + } + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root could not be read")} + } + sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() }) + for _, entry := range entries { + if entry.IsDir() || !matchesImportFilename(entry.Name(), policy.pattern) { + continue + } + accessPath := access.join(root.accessPath, entry.Name()) + candidate := selectedImportCandidate{ + path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: root.site, + } + if _, exists := seenCandidates[candidate.path]; exists { + continue + } + seenCandidates[candidate.path] = struct{}{} + discovered = append(discovered, candidate) + } + } + sort.Slice(discovered, func(left, right int) bool { return discovered[left].path < discovered[right].path }) + selected := make([]selectedImportCandidate, 0, len(discovered)) + for _, candidate := range discovered { + missing, isDirectory, diagnostic := validateNamedImportTarget(access, candidate, false, true) + if diagnostic != nil { + return importSelection{}, []Diagnostic{*diagnostic} + } + if missing || isDirectory || excludedBasename(path.Base(candidate.path), policy.exclude) { + continue + } + selected = append(selected, candidate) + } + if policy.required && len(selected) == 0 { + return importSelection{}, []Diagnostic{*importSelectionDiagnostic(spec.fieldSites["dir"], CodeImportMissing, "required directory import selected no files")} + } + selection := importSelection{candidates: selected} + if len(selected) != 0 { + selection.outcome = importSelectionHasCandidates + } + return selection, nil +} + +func normalizedImportCandidatePath(access *filesystemAccess, accessPath string) string { + if access.native != nil { + return accessPath + } + return filepath.ToSlash(filepath.Clean(accessPath)) +} + +func matchesImportFilename(name string, patterns []string) bool { + if patterns == nil { + return true + } + for _, pattern := range patterns { + matched, err := path.Match(pattern, name) + if err == nil && matched { + return true + } + } + return false +} + +func validateNamedImportTarget(access *filesystemAccess, candidate selectedImportCandidate, required, allowDirectory bool) (bool, bool, *Diagnostic) { if !access.contains(candidate.accessPath) { - return false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target is outside the configuration root") + return false, false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target is outside the configuration root") } canonical, err := access.canonical(candidate.accessPath) if err != nil { if errors.Is(err, fs.ErrNotExist) { if required { - return false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist") + return false, false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist") } - return true, nil + return true, false, nil } if errors.Is(err, fs.ErrPermission) { - return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed") + return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed") } - return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be canonicalized") + return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be canonicalized") } if !access.contains(canonical) { - return false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target resolves outside the configuration root") + return false, false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target resolves outside the configuration root") } info, err := access.stat(canonical) if err != nil { if errors.Is(err, fs.ErrNotExist) { if required { - return false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist") + return false, false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist") } - return true, nil + return true, false, nil } if errors.Is(err, fs.ErrPermission) { - return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed") + return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed") } - return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be inspected") + return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be inspected") + } + if info.IsDir() && allowDirectory { + return false, true, nil } if !info.Mode().IsRegular() { - return false, importSelectionDiagnostic(candidate.site, CodeSourceNotRegular, "source is not a regular file") + return false, false, importSelectionDiagnostic(candidate.site, CodeSourceNotRegular, "source is not a regular file") } - return false, nil + return false, false, nil } func excludedBasename(name string, patterns []string) bool { diff --git a/internal/engine/import_spec.go b/internal/engine/import_spec.go index cfc28a7..f1c5c78 100644 --- a/internal/engine/import_spec.go +++ b/internal/engine/import_spec.go @@ -21,26 +21,26 @@ type importFieldSite struct { // importDeclarationSpec preserves authored presence until discovery policy is resolved. type importDeclarationSpec struct { - path Path - location Location - selector importSelector - paths []string - pathSites []importFieldSite - glob string - globSegments []string - required *bool - recursive *bool - follow *bool - hidden *bool - pattern []string - patternSet bool - exclude []string - excludeSet bool - depth int - depthSet bool - destination Path - placement placementMode - fieldSites map[string]importFieldSite + path Path + location Location + selector importSelector + paths []string + selectorSites []importFieldSite + glob string + globSegments []string + required *bool + recursive *bool + follow *bool + hidden *bool + pattern []string + patternSet bool + exclude []string + excludeSet bool + depth int + depthSet bool + destination Path + placement placementMode + fieldSites map[string]importFieldSite } // importDefaults are the engine's private copy of load-wide discovery defaults. @@ -287,7 +287,7 @@ func parseImportDeclarationSpec(node *document.Node, declarationPath Path) (impo return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, selectorEntry.Key+" contains an invalid source path", badNode)} } badPath, badNode := listMemberSite(selectorEntry.Value, selectorPath, origins[index]) - spec.pathSites = append(spec.pathSites, importFieldSite{path: badPath.clone(), location: publicLocation(badNode.Location)}) + spec.selectorSites = append(spec.selectorSites, importFieldSite{path: badPath.clone(), location: publicLocation(badNode.Location)}) } spec.paths = values case importSelectorGlob: