Expand ordered configuration search candidates

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Configuration Discovery Implementation Plan
 
-Status: in progress. Stage 1 is complete.
+Status: in progress. Stages 1 and 2 are complete.
 
 Implement the target behavior in [the discovery roadmap](discovery.md). That
 document owns the selected conventions and OD-1 through OD-5; this plan owns the
@@ -220,7 +220,7 @@ Acceptance: descriptions and validation are testable without acquisition; the
 public loading behavior is unchanged. Run the new focused engine tests and the
 everyday check. Mark and commit the stage.
 
-## Stage 2 — Expand Ordered Candidates And Resolve Preset Bases
+## Stage 2 — Expand Ordered Candidates And Resolve Preset Bases (Complete)
 
 1. Implement incremental ordered expansion for literal paths, directory entries,
    groups, and presets. Preserve component positions and filename order exactly.

internal/engine/discovery.go

@@ -1,6 +1,8 @@
 package engine
 
 import (
+	"errors"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -264,3 +266,196 @@ func validWindowsFilenameComponent(value string) bool {
 		return true
 	}
 }
+
+var errUnixSystemPresetUnavailable = errors.New("Unix system directories are unavailable on this platform")
+
+type discoveryDirectoryHelpers struct {
+	userConfigDir func() (string, error)
+	userHomeDir   func() (string, error)
+	workingDir    func() (string, error)
+	unixSupported func() bool
+}
+
+type searchCandidateIterator struct {
+	search      Search
+	application string
+	helpers     discoveryDirectoryHelpers
+	entryIndex  int
+	groupIndex  int
+	nameIndex   int
+	directory   string
+	dirReady    bool
+}
+
+func newSearchCandidateIterator(search Search, application string, helpers discoveryDirectoryHelpers) (*searchCandidateIterator, []Diagnostic) {
+	if diagnostics := validateSearch(search, application); len(diagnostics) != 0 {
+		return nil, diagnostics
+	}
+	defaults := defaultDiscoveryDirectoryHelpers()
+	if helpers.userConfigDir == nil {
+		helpers.userConfigDir = defaults.userConfigDir
+	}
+	if helpers.userHomeDir == nil {
+		helpers.userHomeDir = defaults.userHomeDir
+	}
+	if helpers.workingDir == nil {
+		helpers.workingDir = defaults.workingDir
+	}
+	if helpers.unixSupported == nil {
+		helpers.unixSupported = defaults.unixSupported
+	}
+	return &searchCandidateIterator{search: search, application: application, helpers: helpers}, nil
+}
+
+func defaultDiscoveryDirectoryHelpers() discoveryDirectoryHelpers {
+	return discoveryDirectoryHelpers{
+		userConfigDir: os.UserConfigDir,
+		userHomeDir:   os.UserHomeDir,
+		workingDir:    os.Getwd,
+		unixSupported: unixSystemPresetSupported,
+	}
+}
+
+// next expands one candidate at a time, resolving only the directory currently
+// reached. Directory filenames are visited before moving to the next directory.
+func (iterator *searchCandidateIterator) next() (string, bool, error) {
+	for {
+		if iterator.dirReady {
+			entry := iterator.search.entries[iterator.entryIndex]
+			for iterator.groupIndex < len(entry.filenames) {
+				group := entry.filenames[iterator.groupIndex]
+				count := filenameGroupLength(group)
+				if iterator.nameIndex < count {
+					name := filenameGroupAt(group, iterator.nameIndex, iterator.application)
+					iterator.nameIndex++
+					return filepath.Join(iterator.directory, name), true, nil
+				}
+				iterator.groupIndex++
+				iterator.nameIndex = 0
+			}
+			iterator.dirReady = false
+			iterator.entryIndex++
+		}
+		if iterator.entryIndex >= len(iterator.search.entries) {
+			return "", false, nil
+		}
+		entry := iterator.search.entries[iterator.entryIndex]
+		if entry.kind == searchEntryPath {
+			iterator.entryIndex++
+			path, err := iterator.absoluteCandidate(entry.path)
+			if err != nil {
+				return "", false, err
+			}
+			return path, true, nil
+		}
+		if !directoryHasCandidates(entry.filenames) {
+			iterator.entryIndex++
+			continue
+		}
+		directory, err := iterator.resolveDirectory(entry)
+		if err != nil {
+			return "", false, err
+		}
+		iterator.directory = directory
+		iterator.groupIndex = 0
+		iterator.nameIndex = 0
+		iterator.dirReady = true
+	}
+}
+
+func (iterator *searchCandidateIterator) absoluteCandidate(path string) (string, error) {
+	if filepath.IsAbs(path) {
+		return filepath.Clean(path), nil
+	}
+	if filepath.VolumeName(path) != "" {
+		return filepath.Abs(path)
+	}
+	working, err := iterator.helpers.workingDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Clean(filepath.Join(working, path)), nil
+}
+
+func (iterator *searchCandidateIterator) resolveDirectory(entry searchEntry) (string, error) {
+	var base string
+	switch entry.baseKind {
+	case directoryBaseLiteral:
+		base = entry.directory
+	case directoryBaseUnixLocal:
+		if !iterator.helpers.unixSupported() {
+			return "", errUnixSystemPresetUnavailable
+		}
+		base = "/usr/local/etc"
+	case directoryBaseUnixEtc:
+		if !iterator.helpers.unixSupported() {
+			return "", errUnixSystemPresetUnavailable
+		}
+		base = "/etc"
+	case directoryBaseUserConfig:
+		var err error
+		base, err = iterator.helpers.userConfigDir()
+		if err != nil {
+			return "", err
+		}
+	case directoryBaseHome:
+		var err error
+		base, err = iterator.helpers.userHomeDir()
+		if err != nil {
+			return "", err
+		}
+	case directoryBaseWorking:
+		var err error
+		base, err = iterator.helpers.workingDir()
+		if err != nil {
+			return "", err
+		}
+	default:
+		return "", errors.New("directory search base is invalid")
+	}
+	if entry.appendSlug {
+		base = filepath.Join(base, entry.slugPrefix+iterator.application)
+	}
+	if entry.baseKind == directoryBaseLiteral && !filepath.IsAbs(base) {
+		return iterator.absoluteCandidate(base)
+	}
+	return filepath.Clean(base), nil
+}
+
+func directoryHasCandidates(groups []FilenameGroup) bool {
+	for _, group := range groups {
+		if filenameGroupLength(group) != 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func filenameGroupLength(group FilenameGroup) int {
+	switch group.kind {
+	case filenameGroupNamed:
+		return 2
+	case filenameGroupGeneric:
+		return 4
+	case filenameGroupCustom:
+		return len(group.names)
+	default:
+		return 0
+	}
+}
+
+func filenameGroupAt(group FilenameGroup, index int, application string) string {
+	switch group.kind {
+	case filenameGroupNamed:
+		if index == 0 {
+			return application + ".yml"
+		}
+		return application + ".yaml"
+	case filenameGroupGeneric:
+		return [...]string{"config.yml", "config.yaml", "main.yml", "main.yaml"}[index]
+	case filenameGroupCustom:
+		return group.names[index]
+	default:
+		return ""
+	}
+}

internal/engine/discovery_platform_other.go

@@ -0,0 +1,5 @@
+//go:build !aix && !android && !darwin && !dragonfly && !freebsd && !hurd && !illumos && !ios && !linux && !netbsd && !openbsd && !solaris
+
+package engine
+
+func unixSystemPresetSupported() bool { return false }

internal/engine/discovery_platform_unix.go

@@ -0,0 +1,5 @@
+//go:build aix || android || darwin || dragonfly || freebsd || hurd || illumos || ios || linux || netbsd || openbsd || solaris
+
+package engine
+
+func unixSystemPresetSupported() bool { return true }

internal/engine/discovery_test.go

@@ -1,6 +1,8 @@
 package engine
 
 import (
+	"errors"
+	"path/filepath"
 	"runtime"
 	"testing"
 )
@@ -150,3 +152,184 @@ func TestFilenameComponentValidation(t *testing.T) {
 		}
 	}
 }
+
+func TestSearchCandidatesPreserveDirectoryAndGroupOrder(t *testing.T) {
+	t.Parallel()
+
+	root := t.TempDir()
+	helpers := discoveryDirectoryHelpers{workingDir: func() (string, error) { return root, nil }}
+	search := Searches(
+		Paths("first.yml"),
+		SearchDirectory("settings", DirectorySearchOptions{Filenames: []FilenameGroup{
+			GenericFilenames(), CustomFilenames("special.yml"),
+		}}),
+		Paths("last.yml"),
+	)
+	candidates := collectSearchCandidates(t, search, "app", helpers)
+	want := []string{
+		filepath.Join(root, "first.yml"),
+		filepath.Join(root, "settings", "config.yml"),
+		filepath.Join(root, "settings", "config.yaml"),
+		filepath.Join(root, "settings", "main.yml"),
+		filepath.Join(root, "settings", "main.yaml"),
+		filepath.Join(root, "settings", "special.yml"),
+		filepath.Join(root, "last.yml"),
+	}
+	assertCandidatePaths(t, candidates, want)
+
+	reversed := SearchDirectory("settings", DirectorySearchOptions{Filenames: []FilenameGroup{NamedFilenames(), GenericFilenames()}})
+	reversedCandidates := collectSearchCandidates(t, reversed, "app", helpers)
+	assertCandidatePaths(t, reversedCandidates[:2], []string{
+		filepath.Join(root, "settings", "app.yml"),
+		filepath.Join(root, "settings", "app.yaml"),
+	})
+}
+
+func TestSearchPresetCandidateLists(t *testing.T) {
+	t.Parallel()
+
+	helpers := discoveryDirectoryHelpers{
+		userConfigDir: func() (string, error) { return "/controlled/config", nil },
+		userHomeDir:   func() (string, error) { return "/controlled/home", nil },
+		workingDir:    func() (string, error) { return "/controlled/work", nil },
+		unixSupported: func() bool { return true },
+	}
+
+	unix := collectSearchCandidates(t, UnixSystemDirectories(), "app", helpers)
+	if len(unix) != 14 {
+		t.Fatalf("Unix preset candidate count = %d: %#v", len(unix), unix)
+	}
+	assertCandidatePaths(t, unix, []string{
+		"/usr/local/etc/app/app.yml", "/usr/local/etc/app/app.yaml", "/usr/local/etc/app/config.yml", "/usr/local/etc/app/config.yaml", "/usr/local/etc/app/main.yml", "/usr/local/etc/app/main.yaml",
+		"/etc/app/app.yml", "/etc/app/app.yaml", "/etc/app/config.yml", "/etc/app/config.yaml", "/etc/app/main.yml", "/etc/app/main.yaml",
+		"/etc/app.yml", "/etc/app.yaml",
+	})
+
+	user := collectSearchCandidates(t, UserConfigDirectory(), "app", helpers)
+	assertCandidatePaths(t, user, []string{
+		"/controlled/config/app/app.yml", "/controlled/config/app/app.yaml", "/controlled/config/app/config.yml", "/controlled/config/app/config.yaml", "/controlled/config/app/main.yml", "/controlled/config/app/main.yaml",
+		"/controlled/config/app.yml", "/controlled/config/app.yaml",
+	})
+	home := collectSearchCandidates(t, HomeDirectory(), "app", helpers)
+	assertCandidatePaths(t, home, []string{
+		"/controlled/home/.app/app.yml", "/controlled/home/.app/app.yaml", "/controlled/home/.app/config.yml", "/controlled/home/.app/config.yaml", "/controlled/home/.app/main.yml", "/controlled/home/.app/main.yaml",
+		"/controlled/home/app.yml", "/controlled/home/app.yaml",
+	})
+	working := collectSearchCandidates(t, WorkingDirectory(), "app", helpers)
+	assertCandidatePaths(t, working, []string{"/controlled/work/app.yml", "/controlled/work/app.yaml"})
+}
+
+func TestSearchCandidateExpansionIsLazyAndSkipsEmptyDirectories(t *testing.T) {
+	t.Parallel()
+
+	configCalls := 0
+	helpers := discoveryDirectoryHelpers{
+		userConfigDir: func() (string, error) {
+			configCalls++
+			return "/controlled/config", nil
+		},
+		workingDir:    func() (string, error) { return "/controlled/work", nil },
+		unixSupported: func() bool { return true },
+	}
+	search := Searches(Paths("/selected.yml"), UserConfigDirectory())
+	iterator, diagnostics := newSearchCandidateIterator(search, "app", helpers)
+	if len(diagnostics) != 0 {
+		t.Fatalf("iterator diagnostics = %#v", diagnostics)
+	}
+	if candidate, ok, err := iterator.next(); err != nil || !ok || candidate != "/selected.yml" {
+		t.Fatalf("first candidate = %q, %v, %v", candidate, ok, err)
+	}
+	if configCalls != 0 {
+		t.Fatalf("later user directory resolved before consumption: %d calls", configCalls)
+	}
+
+	emptyOnly := SearchDirectory("should-not-resolve", DirectorySearchOptions{Filenames: []FilenameGroup{}})
+	emptyCandidates := collectSearchCandidates(t, emptyOnly, "", helpers)
+	if len(emptyCandidates) != 0 || configCalls != 0 {
+		t.Fatalf("empty directory candidates = %#v, helper calls = %d", emptyCandidates, configCalls)
+	}
+	continued := collectSearchCandidates(t, Searches(emptyOnly, Paths("later.yml")), "", helpers)
+	assertCandidatePaths(t, continued, []string{filepath.Join("/controlled/work", "later.yml")})
+	if configCalls != 0 {
+		t.Fatalf("empty directory base was resolved: %d calls", configCalls)
+	}
+}
+
+func TestSearchCandidateBaseFailuresAndPlatformApplicability(t *testing.T) {
+	t.Parallel()
+
+	baseFailure := errors.New("controlled base failure")
+	helpers := discoveryDirectoryHelpers{
+		userConfigDir: func() (string, error) { return "", baseFailure },
+		workingDir:    func() (string, error) { return "/controlled/work", nil },
+		unixSupported: func() bool { return false },
+	}
+	iterator, diagnostics := newSearchCandidateIterator(UserConfigDirectory(), "app", helpers)
+	if len(diagnostics) != 0 {
+		t.Fatalf("user preset validation = %#v", diagnostics)
+	}
+	if _, ok, err := iterator.next(); ok || !errors.Is(err, baseFailure) {
+		t.Fatalf("unavailable user base = ok %v, err %v", ok, err)
+	}
+
+	iterator, diagnostics = newSearchCandidateIterator(UnixSystemDirectories(), "app", helpers)
+	if len(diagnostics) != 0 {
+		t.Fatalf("Unix preset validation = %#v", diagnostics)
+	}
+	if _, ok, err := iterator.next(); ok || !errors.Is(err, errUnixSystemPresetUnavailable) {
+		t.Fatalf("unsupported Unix preset = ok %v, err %v", ok, err)
+	}
+
+	iterator, diagnostics = newSearchCandidateIterator(Searches(Paths("/earlier.yml"), UnixSystemDirectories()), "app", helpers)
+	if len(diagnostics) != 0 {
+		t.Fatalf("composed Unix preset validation = %#v", diagnostics)
+	}
+	if candidate, ok, err := iterator.next(); err != nil || !ok || candidate != "/earlier.yml" {
+		t.Fatalf("earlier candidate = %q, %v, %v", candidate, ok, err)
+	}
+}
+
+func TestRelativeAndShellLikeSearchPathsRemainLiteral(t *testing.T) {
+	t.Parallel()
+
+	root := t.TempDir()
+	helpers := discoveryDirectoryHelpers{workingDir: func() (string, error) { return root, nil }}
+	search := Searches(Paths("~/literal.yml", "$CONFIG/*.yaml"), SearchDirectory("./settings", DirectorySearchOptions{Filenames: []FilenameGroup{CustomFilenames("name with spaces.yml")}}))
+	candidates := collectSearchCandidates(t, search, "", helpers)
+	assertCandidatePaths(t, candidates, []string{
+		filepath.Join(root, "~/literal.yml"),
+		filepath.Join(root, "$CONFIG/*.yaml"),
+		filepath.Join(root, "settings", "name with spaces.yml"),
+	})
+}
+
+func collectSearchCandidates(t *testing.T, search Search, application string, helpers discoveryDirectoryHelpers) []string {
+	t.Helper()
+	iterator, diagnostics := newSearchCandidateIterator(search, application, helpers)
+	if len(diagnostics) != 0 {
+		t.Fatalf("search validation = %#v", diagnostics)
+	}
+	var candidates []string
+	for {
+		candidate, ok, err := iterator.next()
+		if err != nil {
+			t.Fatalf("expand candidate: %v", err)
+		}
+		if !ok {
+			return candidates
+		}
+		candidates = append(candidates, candidate)
+	}
+}
+
+func assertCandidatePaths(t *testing.T, got, want []string) {
+	t.Helper()
+	if len(got) != len(want) {
+		t.Fatalf("candidate count = %d, want %d: %#v", len(got), len(want), got)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("candidate %d = %q, want %q", i, got[i], want[i])
+		}
+	}
+}