Select shallow directory import sources

internal/engine/directory_selection_test.go

@@ -0,0 +1,228 @@
+package engine
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+	"testing/fstest"
+)
+
+func TestSelectShallowDirectoriesUnionsRootsAndPatterns(t *testing.T) {
+	t.Parallel()
+
+	filesystem := fstest.MapFS{
+		"one":                  &fstest.MapFile{Mode: fs.ModeDir | 0o700},
+		"one/a.yaml":           &fstest.MapFile{Data: []byte("a: 1\n")},
+		"one/README":           &fstest.MapFile{Data: []byte("readme\n")},
+		"one/.hidden":          &fstest.MapFile{Data: []byte("hidden\n")},
+		"one/draft.draft.yaml": &fstest.MapFile{Data: []byte("draft\n")},
+		"one/nested/deep.yaml": &fstest.MapFile{Data: []byte("deep: 1\n")},
+		"two":                  &fstest.MapFile{Mode: fs.ModeDir | 0o700},
+		"two/a.yaml":           &fstest.MapFile{Data: []byte("a: 2\n")},
+		"three":                &fstest.MapFile{Mode: fs.ModeDir | 0o700},
+		"three/extensionless":  &fstest.MapFile{Data: []byte("plain\n")},
+	}
+	access := newNativeAccess(filesystem, ".", &namespaceToken{})
+	specs, diagnostics := parseImportSpecsYAML(t, `
+    - dir: [one, one, missing, two]
+      pattern: ['*.yaml', 'a.*', 'README']
+      exclude: '*.draft.yaml'
+      required: true
+`)
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	accounting := &importAccounting{}
+	selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), accounting)
+	if len(diagnostics) != 0 {
+		t.Fatalf("selection diagnostics = %#v", diagnostics)
+	}
+	if selection.outcome != importSelectionHasCandidates || len(selection.candidates) != 3 {
+		t.Fatalf("selection = %#v, want three unioned matches", selection)
+	}
+	if selection.candidates[0].path != "one/README" || selection.candidates[1].path != "one/a.yaml" || selection.candidates[2].path != "two/a.yaml" {
+		t.Fatalf("selected paths = %#v", selection.candidates)
+	}
+	if accounting.entries != 6 {
+		t.Fatalf("entry visits = %d, want 6 entries across distinct roots", accounting.entries)
+	}
+}
+
+func TestSelectShallowDirectoriesIncludeUnrestrictedHiddenAndExtensionlessFiles(t *testing.T) {
+	t.Parallel()
+
+	root := t.TempDir()
+	for name, value := range map[string]string{
+		"a.yaml":      "a: 1\n",
+		".private":    "private\n",
+		"README":      "readme\n",
+		"nested/deep": "deep\n",
+	} {
+		filename := filepath.Join(root, filepath.FromSlash(name))
+		if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filename, []byte(value), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	access, err := openHostAccess(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer access.close()
+
+	specs, diagnostics := parseImportSpecsYAML(t, "    - dir: .\n")
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
+	if len(diagnostics) != 0 {
+		t.Fatalf("selection diagnostics = %#v", diagnostics)
+	}
+	if len(selection.candidates) != 3 {
+		t.Fatalf("selected files = %#v, want yaml, hidden, and extensionless files", selection.candidates)
+	}
+	for _, candidate := range selection.candidates {
+		if filepath.Base(candidate.path) == "deep" {
+			t.Fatalf("shallow search descended into a child directory: %#v", selection.candidates)
+		}
+	}
+}
+
+func TestSelectShallowDirectoriesApplyRequirednessToFinalUnion(t *testing.T) {
+	t.Parallel()
+
+	filesystem := fstest.MapFS{
+		"available":               &fstest.MapFile{Mode: fs.ModeDir | 0o700},
+		"available/match.yaml":    &fstest.MapFile{Data: []byte("match: true\n")},
+		"available/excluded.yaml": &fstest.MapFile{Data: []byte("excluded: true\n")},
+	}
+	access := newNativeAccess(filesystem, ".", &namespaceToken{})
+	for _, test := range []struct {
+		name      string
+		yaml      string
+		wantError bool
+		wantCount int
+	}{
+		{
+			name:      "missing root alongside a match",
+			yaml:      "    - dir: [missing, available]\n      pattern: '*.yaml'\n      exclude: excluded.yaml\n      required: true\n",
+			wantCount: 1,
+		},
+		{
+			name:      "all matches excluded",
+			yaml:      "    - dir: available\n      pattern: '*.yaml'\n      exclude: '*.yaml'\n      required: true\n",
+			wantError: true,
+		},
+		{
+			name:      "explicit empty patterns select nothing",
+			yaml:      "    - dir: available\n      pattern: []\n",
+			wantCount: 0,
+		},
+		{
+			name:      "empty search roots",
+			yaml:      "    - dir: []\n      required: true\n",
+			wantError: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			specs, diagnostics := parseImportSpecsYAML(t, test.yaml)
+			if len(diagnostics) != 0 {
+				t.Fatal(diagnostics)
+			}
+			policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
+			if len(diagnostics) != 0 {
+				t.Fatal(diagnostics)
+			}
+			selection, diagnostics := selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
+			if test.wantError {
+				if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportMissing {
+					t.Fatalf("diagnostics = %#v, want required empty-union error", diagnostics)
+				}
+				return
+			}
+			if len(diagnostics) != 0 || len(selection.candidates) != test.wantCount {
+				t.Fatalf("selection, diagnostics = %#v, %#v", selection, diagnostics)
+			}
+			if test.wantCount == 1 && selection.candidates[0].path != "available/match.yaml" {
+				t.Fatalf("selected path = %q, want available/match.yaml", selection.candidates[0].path)
+			}
+		})
+	}
+}
+
+func TestSelectShallowDirectoryRejectsInvalidRootInsteadOfTreatingItAsMissing(t *testing.T) {
+	t.Parallel()
+
+	filesystem := fstest.MapFS{
+		"file.yaml": &fstest.MapFile{Data: []byte("not a directory\n")},
+	}
+	access := newNativeAccess(filesystem, ".", &namespaceToken{})
+	specs, diagnostics := parseImportSpecsYAML(t, "    - dir: file.yaml\n")
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	_, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
+	if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
+		t.Fatalf("invalid-root diagnostics = %#v", diagnostics)
+	}
+
+	denied := &statOnlyImportFilesystem{statErrors: map[string]error{"denied": fs.ErrPermission}}
+	access = newNativeAccess(denied, ".", &namespaceToken{})
+	specs, diagnostics = parseImportSpecsYAML(t, "    - dir: denied\n")
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	policy, diagnostics = resolveImportPolicy(specs[0], importDefaults{})
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	_, diagnostics = selectShallowDirectoryImportCandidates(access, ".", specs[0], policy, defaultImportLimits(), &importAccounting{})
+	if len(diagnostics) != 1 || diagnostics[0].Code != CodeImportInvalid {
+		t.Fatalf("inaccessible-root diagnostics = %#v", diagnostics)
+	}
+}
+
+func TestSelectShallowDirectoryFiltersSymlinkAliasSpelling(t *testing.T) {
+	t.Parallel()
+
+	root := t.TempDir()
+	if err := os.WriteFile(filepath.Join(root, "welcome.yaml"), []byte("welcome: true\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink("welcome.yaml", filepath.Join(root, "welcome.draft.yaml")); err != nil {
+		t.Skipf("symlinks are unavailable: %v", err)
+	}
+	access, err := openHostAccess(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer access.close()
+
+	specs, diagnostics := parseImportSpecsYAML(t, "    - dir: .\n      pattern: '*.yaml'\n      exclude: '*.draft.yaml'\n")
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	policy, diagnostics := resolveImportPolicy(specs[0], importDefaults{})
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	selection, diagnostics := selectShallowDirectoryImportCandidates(access, root, specs[0], policy, defaultImportLimits(), &importAccounting{})
+	if len(diagnostics) != 0 || len(selection.candidates) != 1 || filepath.Base(selection.candidates[0].path) != "welcome.yaml" {
+		t.Fatalf("alias filtering selection, diagnostics = %#v, %#v", selection, diagnostics)
+	}
+}

internal/engine/import_selection.go

@@ -41,8 +41,8 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec
 		}
 		seen[candidatePath] = struct{}{}
 		var site importFieldSite
-		if index < len(spec.pathSites) {
+		if index < len(spec.selectorSites) {
-			site = spec.pathSites[index]
+			site = spec.selectorSites[index]
 		} else {
 			site = spec.fieldSites["path"]
 		}
@@ -53,7 +53,7 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec
 	})
 	candidates := make([]selectedImportCandidate, 0, len(normalized))
 	for _, candidate := range normalized {
-		missing, diagnostic := validateNamedImportTarget(access, candidate, policy.required)
+		missing, _, diagnostic := validateNamedImportTarget(access, candidate, policy.required, false)
 		if diagnostic != nil {
 			return importSelection{}, []Diagnostic{*diagnostic}
 		}
@@ -72,43 +72,159 @@ func selectNamedImportCandidates(access *filesystemAccess, basePath string, spec
 	return selection, nil
 }
 
-func validateNamedImportTarget(access *filesystemAccess, candidate selectedImportCandidate, required bool) (bool, *Diagnostic) {
+func selectShallowDirectoryImportCandidates(access *filesystemAccess, basePath string, spec importDeclarationSpec, policy resolvedImportPolicy, limits importLimits, accounting *importAccounting) (importSelection, []Diagnostic) {
+	roots := make([]selectedImportCandidate, 0, len(spec.paths))
+	seenRoots := make(map[string]struct{}, len(spec.paths))
+	for index, relative := range spec.paths {
+		accessPath := access.join(basePath, relative)
+		candidatePath := normalizedImportCandidatePath(access, accessPath)
+		if _, exists := seenRoots[candidatePath]; exists {
+			continue
+		}
+		seenRoots[candidatePath] = struct{}{}
+		site := spec.fieldSites["dir"]
+		if index < len(spec.selectorSites) {
+			site = spec.selectorSites[index]
+		}
+		roots = append(roots, selectedImportCandidate{path: candidatePath, accessPath: accessPath, site: site})
+	}
+	sort.Slice(roots, func(left, right int) bool { return roots[left].path < roots[right].path })
+
+	if accounting == nil {
+		accounting = &importAccounting{}
+	}
+	discovered := make([]selectedImportCandidate, 0)
+	seenCandidates := make(map[string]struct{})
+	for _, root := range roots {
+		missing, isDirectory, diagnostic := validateNamedImportTarget(access, root, false, true)
+		if diagnostic != nil {
+			return importSelection{}, []Diagnostic{*diagnostic}
+		}
+		if missing {
+			continue
+		}
+		if !isDirectory {
+			return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root is not a directory")}
+		}
+
+		entries := make([]fs.DirEntry, 0)
+		var problem *Diagnostic
+		err := access.readDirectory(root.accessPath, func(entry fs.DirEntry) bool {
+			accounting.entries++
+			if accounting.entries > limits.entries {
+				problem = importSelectionDiagnostic(root.site, CodeImportLimitExceeded, "directory search exceeds the directory-entry visit limit")
+				return false
+			}
+			entries = append(entries, entry)
+			return true
+		})
+		if problem != nil {
+			return importSelection{}, []Diagnostic{*problem}
+		}
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return importSelection{}, []Diagnostic{*importSelectionDiagnostic(root.site, CodeImportInvalid, "directory search root could not be read")}
+		}
+		sort.Slice(entries, func(left, right int) bool { return entries[left].Name() < entries[right].Name() })
+		for _, entry := range entries {
+			if entry.IsDir() || !matchesImportFilename(entry.Name(), policy.pattern) {
+				continue
+			}
+			accessPath := access.join(root.accessPath, entry.Name())
+			candidate := selectedImportCandidate{
+				path: normalizedImportCandidatePath(access, accessPath), accessPath: accessPath, site: root.site,
+			}
+			if _, exists := seenCandidates[candidate.path]; exists {
+				continue
+			}
+			seenCandidates[candidate.path] = struct{}{}
+			discovered = append(discovered, candidate)
+		}
+	}
+	sort.Slice(discovered, func(left, right int) bool { return discovered[left].path < discovered[right].path })
+	selected := make([]selectedImportCandidate, 0, len(discovered))
+	for _, candidate := range discovered {
+		missing, isDirectory, diagnostic := validateNamedImportTarget(access, candidate, false, true)
+		if diagnostic != nil {
+			return importSelection{}, []Diagnostic{*diagnostic}
+		}
+		if missing || isDirectory || excludedBasename(path.Base(candidate.path), policy.exclude) {
+			continue
+		}
+		selected = append(selected, candidate)
+	}
+	if policy.required && len(selected) == 0 {
+		return importSelection{}, []Diagnostic{*importSelectionDiagnostic(spec.fieldSites["dir"], CodeImportMissing, "required directory import selected no files")}
+	}
+	selection := importSelection{candidates: selected}
+	if len(selected) != 0 {
+		selection.outcome = importSelectionHasCandidates
+	}
+	return selection, nil
+}
+
+func normalizedImportCandidatePath(access *filesystemAccess, accessPath string) string {
+	if access.native != nil {
+		return accessPath
+	}
+	return filepath.ToSlash(filepath.Clean(accessPath))
+}
+
+func matchesImportFilename(name string, patterns []string) bool {
+	if patterns == nil {
+		return true
+	}
+	for _, pattern := range patterns {
+		matched, err := path.Match(pattern, name)
+		if err == nil && matched {
+			return true
+		}
+	}
+	return false
+}
+
+func validateNamedImportTarget(access *filesystemAccess, candidate selectedImportCandidate, required, allowDirectory bool) (bool, bool, *Diagnostic) {
 	if !access.contains(candidate.accessPath) {
-		return false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target is outside the configuration root")
+		return false, false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target is outside the configuration root")
 	}
 	canonical, err := access.canonical(candidate.accessPath)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			if required {
-				return false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
+				return false, false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
 			}
-			return true, nil
+			return true, false, nil
 		}
 		if errors.Is(err, fs.ErrPermission) {
-			return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
+			return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
 		}
-		return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be canonicalized")
+		return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be canonicalized")
 	}
 	if !access.contains(canonical) {
-		return false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target resolves outside the configuration root")
+		return false, false, importSelectionDiagnostic(candidate.site, CodeImportOutsideRoot, "import target resolves outside the configuration root")
 	}
 	info, err := access.stat(canonical)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			if required {
-				return false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
+				return false, false, importSelectionDiagnostic(candidate.site, CodeImportMissing, "required import target does not exist")
 			}
-			return true, nil
+			return true, false, nil
 		}
 		if errors.Is(err, fs.ErrPermission) {
-			return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
+			return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target cannot be accessed")
 		}
-		return false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be inspected")
+		return false, false, importSelectionDiagnostic(candidate.site, CodeImportInvalid, "import target could not be inspected")
+	}
+	if info.IsDir() && allowDirectory {
+		return false, true, nil
 	}
 	if !info.Mode().IsRegular() {
-		return false, importSelectionDiagnostic(candidate.site, CodeSourceNotRegular, "source is not a regular file")
+		return false, false, importSelectionDiagnostic(candidate.site, CodeSourceNotRegular, "source is not a regular file")
 	}
-	return false, nil
+	return false, false, nil
 }
 
 func excludedBasename(name string, patterns []string) bool {

internal/engine/import_spec.go

@@ -21,26 +21,26 @@ type importFieldSite struct {
 
 // importDeclarationSpec preserves authored presence until discovery policy is resolved.
 type importDeclarationSpec struct {
-	path         Path
+	path          Path
-	location     Location
+	location      Location
-	selector     importSelector
+	selector      importSelector
-	paths        []string
+	paths         []string
-	pathSites    []importFieldSite
+	selectorSites []importFieldSite
-	glob         string
+	glob          string
-	globSegments []string
+	globSegments  []string
-	required     *bool
+	required      *bool
-	recursive    *bool
+	recursive     *bool
-	follow       *bool
+	follow        *bool
-	hidden       *bool
+	hidden        *bool
-	pattern      []string
+	pattern       []string
-	patternSet   bool
+	patternSet    bool
-	exclude      []string
+	exclude       []string
-	excludeSet   bool
+	excludeSet    bool
-	depth        int
+	depth         int
-	depthSet     bool
+	depthSet      bool
-	destination  Path
+	destination   Path
-	placement    placementMode
+	placement     placementMode
-	fieldSites   map[string]importFieldSite
+	fieldSites    map[string]importFieldSite
 }
 
 // importDefaults are the engine's private copy of load-wide discovery defaults.
@@ -287,7 +287,7 @@ func parseImportDeclarationSpec(node *document.Node, declarationPath Path) (impo
 				return importDeclarationSpec{}, []Diagnostic{nodeImportDiagnostic(CodeImportInvalid, badPath, selectorEntry.Key+" contains an invalid source path", badNode)}
 			}
 			badPath, badNode := listMemberSite(selectorEntry.Value, selectorPath, origins[index])
-			spec.pathSites = append(spec.pathSites, importFieldSite{path: badPath.clone(), location: publicLocation(badNode.Location)})
+			spec.selectorSites = append(spec.selectorSites, importFieldSite{path: badPath.clone(), location: publicLocation(badNode.Location)})
 		}
 		spec.paths = values
 	case importSelectorGlob: