Select shallow directory import sources
This commit is contained in:
228
internal/engine/directory_selection_test.go
Normal file
228
internal/engine/directory_selection_test.go
Normal file
@@ -0,0 +1,228 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
)
|
||||
|
||||
func TestSelectShallowDirectoriesUnionsRootsAndPatterns(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
filesystem := fstest.MapFS{
|
||||
"one": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
|
||||
"one/a.yaml": &fstest.MapFile{Data: []byte("a: 1\n")},
|
||||
"one/README": &fstest.MapFile{Data: []byte("readme\n")},
|
||||
"one/.hidden": &fstest.MapFile{Data: []byte("hidden\n")},
|
||||
"one/draft.draft.yaml": &fstest.MapFile{Data: []byte("draft\n")},
|
||||
"one/nested/deep.yaml": &fstest.MapFile{Data: []byte("deep: 1\n")},
|
||||
"two": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
|
||||
"two/a.yaml": &fstest.MapFile{Data: []byte("a: 2\n")},
|
||||
"three": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
|
||||
"three/extensionless": &fstest.MapFile{Data: []byte("plain\n")},
|
||||
}
|
||||
access := newNativeAccess(filesystem, ".", &namespaceToken{})
|
||||
specs, diagnostics := parseImportSpecsYAML(t, `
|
||||
- dir: [one, one, missing, two]
|
||||
pattern: ['*.yaml', 'a.*', 'README']
|
||||
exclude: '*.draft.yaml'
|
||||
required: true
|
||||
`)
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
accounting := &importAccounting{}
|
||||
selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting)
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatalf("selection diagnostics = %#v", diagnostics)
|
||||
}
|
||||
if selection.outcome != importSelectionHasCandidates || len(selection.candidates) != 3 {
|
||||
t.Fatalf("selection = %#v, want three unioned matches", selection)
|
||||
}
|
||||
if selection.candidates[0].path != "one/README" || selection.candidates[1].path != "one/a.yaml" || selection.candidates[2].path != "two/a.yaml" {
|
||||
t.Fatalf("selected paths = %#v", selection.candidates)
|
||||
}
|
||||
if accounting.entries != 6 {
|
||||
t.Fatalf("entry visits = %d, want 6 entries across distinct roots", accounting.entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectShallowDirectoriesIncludeUnrestrictedHiddenAndExtensionlessFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
for name, value := range map[string]string{
|
||||
"a.yaml": "a: 1\n",
|
||||
".private": "private\n",
|
||||
"README": "readme\n",
|
||||
"nested/deep": "deep\n",
|
||||
} {
|
||||
filename := filepath.Join(root, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filename, []byte(value), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
access, err := openHostAccess(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer access.close()
|
||||
|
||||
specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n")
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatalf("selection diagnostics = %#v", diagnostics)
|
||||
}
|
||||
if len(selection.candidates) != 3 {
|
||||
t.Fatalf("selected files = %#v, want yaml, hidden, and extensionless files", selection.candidates)
|
||||
}
|
||||
for _, candidate := range selection.candidates {
|
||||
if filepath.Base(candidate.path) == "deep" {
|
||||
t.Fatalf("shallow search descended into a child directory: %#v", selection.candidates)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectShallowDirectoriesApplyRequirednessToFinalUnion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
filesystem := fstest.MapFS{
|
||||
"available": &fstest.MapFile{Mode: fs.ModeDir | 0o700},
|
||||
"available/match.yaml": &fstest.MapFile{Data: []byte("match: true\n")},
|
||||
"available/excluded.yaml": &fstest.MapFile{Data: []byte("excluded: true\n")},
|
||||
}
|
||||
access := newNativeAccess(filesystem, ".", &namespaceToken{})
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
yaml string
|
||||
wantError bool
|
||||
wantCount int
|
||||
}{
|
||||
{
|
||||
name: "missing root alongside a match",
|
||||
yaml: " - dir: [missing, available]\n pattern: '*.yaml'\n exclude: excluded.yaml\n required: true\n",
|
||||
wantCount: 1,
|
||||
},
|
||||
{
|
||||
name: "all matches excluded",
|
||||
yaml: " - dir: available\n pattern: '*.yaml'\n exclude: '*.yaml'\n required: true\n",
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "explicit empty patterns select nothing",
|
||||
yaml: " - dir: available\n pattern: []\n",
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "empty search roots",
|
||||
yaml: " - dir: []\n required: true\n",
|
||||
wantError: true,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
specs, diagnostics := parseImportSpecsYAML(t, test.yaml)
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
|
||||
if test.wantError {
|
||||
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing {
|
||||
t.Fatalf("diagnostics = %#v, want required empty-union error", diagnostics)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(diagnostics) != 0 || len(selection.candidates) != test.wantCount {
|
||||
t.Fatalf("selection, diagnostics = %#v, %#v", selection, diagnostics)
|
||||
}
|
||||
if test.wantCount == 1 && selection.candidates[0].path != "available/match.yaml" {
|
||||
t.Fatalf("selected path = %q, want available/match.yaml", selection.candidates[0].path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
filesystem := fstest.MapFS{
|
||||
"file.yaml": &fstest.MapFile{Data: []byte("not a directory\n")},
|
||||
}
|
||||
access := newNativeAccess(filesystem, ".", &namespaceToken{})
|
||||
specs, diagnostics := parseImportSpecsYAML(t, " - dir: file.yaml\n")
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
_, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
|
||||
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
|
||||
t.Fatalf("invalid-root diagnostics = %#v", diagnostics)
|
||||
}
|
||||
|
||||
denied := &statOnlyImportFilesystem{statErrors: map[string]error{"denied": fs.ErrPermission}}
|
||||
access = newNativeAccess(denied, ".", &namespaceToken{})
|
||||
specs, diagnostics = parseImportSpecsYAML(t, " - dir: denied\n")
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
policy, diagnostics = resolveImportPolicy(specs[0], importDefaults{})
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
_, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
|
||||
if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
|
||||
t.Fatalf("inaccessible-root diagnostics = %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectShallowDirectoryFiltersSymlinkAliasSpelling(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "welcome.yaml"), []byte("welcome: true\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink("welcome.yaml", filepath.Join(root, "welcome.draft.yaml")); err != nil {
|
||||
t.Skipf("symlinks are unavailable: %v", err)
|
||||
}
|
||||
access, err := openHostAccess(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer access.close()
|
||||
|
||||
specs, diagnostics := parseImportSpecsYAML(t, " - dir: .\n pattern: '*.yaml'\n exclude: '*.draft.yaml'\n")
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
|
||||
if len(diagnostics) != 0 {
|
||||
t.Fatal(diagnostics)
|
||||
}
|
||||
selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
|
||||
if len(diagnostics) != 0 || len(selection.candidates) != 1 || filepath.Base(selection.candidates[0].path) != "welcome.yaml" {
|
||||
t.Fatalf("alias filtering selection, diagnostics = %#v, %#v", selection, diagnostics)
|
||||
}
|
||||
}
|
||||
@@ -41,8 +41,8 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec
|
||||
}
|
||||
seen[candidatePath] = struct{}{}
|
||||
var site importFieldSite
|
||||
if index < len(spec.pathSites) {
|
||||
site = spec.pathSites[index]
|
||||
if index < len(spec.selectorSites) {
|
||||
site = spec.selectorSites[index]
|
||||
} else {
|
||||
site = spec.fieldSites["path"]
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec
|
||||
})
|
||||
candidates := make([]selectedImportCandidate, 0, len(normalized))
|
||||
for _, candidate := range normalized {
|
||||
missing, diagnostic := validateNamedImportTarget(access, candidate, policy.required)
|
||||
missing, _, diagnostic := validateNamedImportTarget(access, candidate, policy.required, false)
|
||||
if diagnostic != nil {
|
||||
return importSelection{}, []Diagnostic{*diagnostic}
|
||||
}
|
||||
@@ -72,43 +72,159 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec
|
||||
return selection, nil
|
||||
}
|
||||
|
||||
func validateNamedImportTarget(access *filesystemAccess, candidate selectedImportCandidate, required bool) (bool, *Diagnostic) {
|
||||
func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) {
|
||||
roots := make([]selectedImportCandidate, 0, len(spec.paths))
|
||||
seenRoots := make(map[string]struct{}, len(spec.paths))
|
||||
for index, relative := range spec.paths {
|
||||
accessPath := access.join(basePath, relative)
|
||||
candidatePath := normalizedImportCandidatePath(access, accessPath)
|
||||
if _, exists := seenRoots[candidatePath]; exists {
|
||||
continue
|
||||
}
|
||||
seenRoots[candidatePath] = struct{}{}
|
||||
site := spec.fieldSites["dir"]
|
||||
if index < len(spec.selectorSites) {
|
||||
site = spec.selectorSites[index]
|
||||
}
|
||||
roots = append(roots, selectedImportCandidate{path: candidatePath, accessPath: accessPath, site: site})
|
||||
}
|
||||
sort.Slice(roots, func(left, right int) bool { return roots[left].path < roots[right].path })
|
||||
|
||||
if accounting == nil {
|
||||
accounting = &importAccounting{}
|
||||
}
|
||||
discovered := make([]selectedImportCandidate, 0)
|
||||
seenCandidates := make(map[string]struct{})
|
||||
for _, root := range roots {
|
||||
missing, isDirectory, diagnostic := validateNamedImportTarget(access, root, false, true)
|
||||
if diagnostic != nil {
|
||||
return importSelection{}, []Diagnostic{*diagnostic}
|
||||
}
|
||||
if missing {
|
||||
continue
|
||||
}
|
||||
if !isDirectory {
|
||||
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root is not a directory")}
|
||||
}
|
||||
|
||||
entries := make([]fs.DirEntry, 0)
|
||||
var problem *Diagnostic
|
||||
err := access.readDirectory(root.accessPath, func(entry fs.DirEntry) bool {
|
||||
accounting.entries++
|
||||
if accounting.entries > limits.entries {
|
||||
problem = importSelectionDiagnostic(root.site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit")
|
||||
return false
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
return true
|
||||
})
|
||||
if problem != nil {
|
||||
return importSelection{}, []Diagnostic{*problem}
|
||||
}
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root could not be read")}
|
||||
}
|
||||
sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() })
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !matchesImportFilename(entry.Name(), policy.pattern) {
|
||||
continue
|
||||
}
|
||||
accessPath := access.join(root.accessPath, entry.Name())
|
||||
candidate := selectedImportCandidate{
|
||||
path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: root.site,
|
||||
}
|
||||
if _, exists := seenCandidates[candidate.path]; exists {
|
||||
continue
|
||||
}
|
||||
seenCandidates[candidate.path] = struct{}{}
|
||||
discovered = append(discovered, candidate)
|
||||
}
|
||||
}
|
||||
sort.Slice(discovered, func(left, right int) bool { return discovered[left].path < discovered[right].path })
|
||||
selected := make([]selectedImportCandidate, 0, len(discovered))
|
||||
for _, candidate := range discovered {
|
||||
missing, isDirectory, diagnostic := validateNamedImportTarget(access, candidate, false, true)
|
||||
if diagnostic != nil {
|
||||
return importSelection{}, []Diagnostic{*diagnostic}
|
||||
}
|
||||
if missing || isDirectory || excludedBasename(path.Base(candidate.path), policy.exclude) {
|
||||
continue
|
||||
}
|
||||
selected = append(selected, candidate)
|
||||
}
|
||||
if policy.required && len(selected) == 0 {
|
||||
return importSelection{}, []Diagnostic{*importSelectionDiagnostic(spec.fieldSites["dir"], CodeImportMissing, "required directory import selected no files")}
|
||||
}
|
||||
selection := importSelection{candidates: selected}
|
||||
if len(selected) != 0 {
|
||||
selection.outcome = importSelectionHasCandidates
|
||||
}
|
||||
return selection, nil
|
||||
}
|
||||
|
||||
func normalizedImportCandidatePath(access *filesystemAccess, accessPath string) string {
|
||||
if access.native != nil {
|
||||
return accessPath
|
||||
}
|
||||
return filepath.ToSlash(filepath.Clean(accessPath))
|
||||
}
|
||||
|
||||
func matchesImportFilename(name string, patterns []string) bool {
|
||||
if patterns == nil {
|
||||
return true
|
||||
}
|
||||
for _, pattern := range patterns {
|
||||
matched, err := path.Match(pattern, name)
|
||||
if err == nil && matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func validateNamedImportTarget(access *filesystemAccess, candidate selectedImportCandidate, required, allowDirectory bool) (bool, bool, *Diagnostic) {
|
||||
if !access.contains(candidate.accessPath) {
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target is outside the configuration root")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target is outside the configuration root")
|
||||
}
|
||||
canonical, err := access.canonical(candidate.accessPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
if required {
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
|
||||
}
|
||||
return true, nil
|
||||
return true, false, nil
|
||||
}
|
||||
if errors.Is(err, fs.ErrPermission) {
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
|
||||
}
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be canonicalized")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be canonicalized")
|
||||
}
|
||||
if !access.contains(canonical) {
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target resolves outside the configuration root")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target resolves outside the configuration root")
|
||||
}
|
||||
info, err := access.stat(canonical)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
if required {
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
|
||||
}
|
||||
return true, nil
|
||||
return true, false, nil
|
||||
}
|
||||
if errors.Is(err, fs.ErrPermission) {
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
|
||||
}
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be inspected")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be inspected")
|
||||
}
|
||||
if info.IsDir() && allowDirectory {
|
||||
return false, true, nil
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return false, importSelectionDiagnostic(candidate.site, CodeSourceNotRegular, "source is not a regular file")
|
||||
return false, false, importSelectionDiagnostic(candidate.site, CodeSourceNotRegular, "source is not a regular file")
|
||||
}
|
||||
return false, nil
|
||||
return false, false, nil
|
||||
}
|
||||
|
||||
func excludedBasename(name string, patterns []string) bool {
|
||||
|
||||
@@ -21,26 +21,26 @@ type importFieldSite struct {
|
||||
|
||||
// importDeclarationSpec preserves authored presence until discovery policy is resolved.
|
||||
type importDeclarationSpec struct {
|
||||
path Path
|
||||
location Location
|
||||
selector importSelector
|
||||
paths []string
|
||||
pathSites []importFieldSite
|
||||
glob string
|
||||
globSegments []string
|
||||
required *bool
|
||||
recursive *bool
|
||||
follow *bool
|
||||
hidden *bool
|
||||
pattern []string
|
||||
patternSet bool
|
||||
exclude []string
|
||||
excludeSet bool
|
||||
depth int
|
||||
depthSet bool
|
||||
destination Path
|
||||
placement placementMode
|
||||
fieldSites map[string]importFieldSite
|
||||
path Path
|
||||
location Location
|
||||
selector importSelector
|
||||
paths []string
|
||||
selectorSites []importFieldSite
|
||||
glob string
|
||||
globSegments []string
|
||||
required *bool
|
||||
recursive *bool
|
||||
follow *bool
|
||||
hidden *bool
|
||||
pattern []string
|
||||
patternSet bool
|
||||
exclude []string
|
||||
excludeSet bool
|
||||
depth int
|
||||
depthSet bool
|
||||
destination Path
|
||||
placement placementMode
|
||||
fieldSites map[string]importFieldSite
|
||||
}
|
||||
|
||||
// importDefaults are the engine's private copy of load-wide discovery defaults.
|
||||
@@ -287,7 +287,7 @@ func parseImportDeclarationSpec(node *document.Node, declarationPath Path) (impo
|
||||
return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, selectorEntry.Key+" contains an invalid source path", badNode)}
|
||||
}
|
||||
badPath, badNode := listMemberSite(selectorEntry.Value, selectorPath, origins[index])
|
||||
spec.pathSites = append(spec.pathSites, importFieldSite{path: badPath.clone(), location: publicLocation(badNode.Location)})
|
||||
spec.selectorSites = append(spec.selectorSites, importFieldSite{path: badPath.clone(), location: publicLocation(badNode.Location)})
|
||||
}
|
||||
spec.paths = values
|
||||
case importSelectorGlob:
|
||||
|
||||
Reference in New Issue
Block a user