Add safe immutable directory promotion

docs/roadmap/implementation.md

@@ -18,7 +18,7 @@ implementation sequence.
 | --- | --- |
 | Stage 1 | Complete |
 | Stage 2 | Complete |
-| Stage 3 | Not started |
+| Stage 3 | Complete |
 | Stage 4 | Not started |
 | Stage 5 | Not started |
 | Stage 6 | Not started |

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.16
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
 	github.com/aws/smithy-go v1.25.1
+	golang.org/x/sys v0.47.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -34,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
 github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
 github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
 github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/fileops/directory.go

@@ -0,0 +1,204 @@
+package fileops
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
+)
+
+const (
+	promotedDirectoryMode = 0o755
+	promotedFileMode      = 0o644
+)
+
+// PromoteDirectory copies an existing regular-file tree into a new directory
+// and installs the complete copy atomically. It never removes the source or
+// replaces an existing destination.
+func PromoteDirectory(src, dst string) error {
+	return promoteDirectory(src, dst, renameDirectoryNoReplace)
+}
+
+func promoteDirectory(src, dst string, install func(string, string) error) error {
+	if strings.TrimSpace(src) == "" || strings.TrimSpace(dst) == "" {
+		return fmt.Errorf("source and destination directory paths are required")
+	}
+
+	sourceInfo, err := os.Lstat(src)
+	if err != nil {
+		return fmt.Errorf("inspect source directory: %w", err)
+	}
+	if !sourceInfo.IsDir() {
+		return fmt.Errorf("source path %q is not a directory", src)
+	}
+
+	if _, err := os.Lstat(dst); err == nil {
+		return fmt.Errorf("destination path %q already exists", dst)
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("inspect destination path: %w", err)
+	}
+
+	insideSource, err := pathWithin(src, dst)
+	if err != nil {
+		return err
+	}
+	if insideSource {
+		return fmt.Errorf("destination path %q must not be inside source directory %q", dst, src)
+	}
+
+	destinationParent := filepath.Dir(dst)
+	parentInfo, err := os.Lstat(destinationParent)
+	if err != nil {
+		return fmt.Errorf("inspect destination parent: %w", err)
+	}
+	if !parentInfo.IsDir() {
+		return fmt.Errorf("destination parent %q is not a directory", destinationParent)
+	}
+
+	temporary, err := os.MkdirTemp(destinationParent, "."+filepath.Base(dst)+".tmp-*")
+	if err != nil {
+		return fmt.Errorf("create temporary destination directory: %w", err)
+	}
+	removeTemporary := true
+	defer func() {
+		if removeTemporary {
+			_ = os.RemoveAll(temporary)
+		}
+	}()
+
+	if err := copyRegularTree(src, temporary); err != nil {
+		return err
+	}
+	if err := os.Chmod(temporary, promotedDirectoryMode); err != nil {
+		return fmt.Errorf("set temporary root permissions: %w", err)
+	}
+	if err := syncDirectory(temporary); err != nil {
+		return fmt.Errorf("sync temporary root: %w", err)
+	}
+	if err := install(temporary, dst); err != nil {
+		return fmt.Errorf("install promoted directory: %w", err)
+	}
+
+	removeTemporary = false
+	_ = syncDirectory(destinationParent)
+	return nil
+}
+
+func copyRegularTree(src, dst string) error {
+	entries, err := os.ReadDir(src)
+	if err != nil {
+		return fmt.Errorf("read source directory %q: %w", src, err)
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].Name() < entries[j].Name()
+	})
+
+	for _, entry := range entries {
+		sourcePath := filepath.Join(src, entry.Name())
+		destinationPath := filepath.Join(dst, entry.Name())
+		info, err := os.Lstat(sourcePath)
+		if err != nil {
+			return fmt.Errorf("inspect source entry %q: %w", sourcePath, err)
+		}
+
+		switch {
+		case info.Mode().IsRegular():
+			if err := copyRegularFile(sourcePath, destinationPath, info); err != nil {
+				return err
+			}
+		case info.IsDir():
+			if err := os.Mkdir(destinationPath, promotedDirectoryMode); err != nil {
+				return fmt.Errorf("create destination directory %q: %w", destinationPath, err)
+			}
+			if err := copyRegularTree(sourcePath, destinationPath); err != nil {
+				return err
+			}
+			if err := os.Chmod(destinationPath, promotedDirectoryMode); err != nil {
+				return fmt.Errorf("set destination directory permissions %q: %w", destinationPath, err)
+			}
+			if err := syncDirectory(destinationPath); err != nil {
+				return fmt.Errorf("sync destination directory %q: %w", destinationPath, err)
+			}
+		default:
+			return fmt.Errorf("source entry %q has unsupported file type %s", sourcePath, info.Mode().Type())
+		}
+	}
+
+	return nil
+}
+
+func copyRegularFile(src, dst string, inspected os.FileInfo) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("open source file %q: %w", src, err)
+	}
+	defer func() { _ = in.Close() }()
+
+	openedInfo, err := in.Stat()
+	if err != nil {
+		return fmt.Errorf("inspect opened source file %q: %w", src, err)
+	}
+	if !openedInfo.Mode().IsRegular() || !os.SameFile(inspected, openedInfo) {
+		return fmt.Errorf("source file %q changed while being copied", src)
+	}
+
+	out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, promotedFileMode)
+	if err != nil {
+		return fmt.Errorf("create destination file %q: %w", dst, err)
+	}
+	closed := false
+	defer func() {
+		if !closed {
+			_ = out.Close()
+		}
+	}()
+
+	if _, err := io.Copy(out, in); err != nil {
+		return fmt.Errorf("copy source file %q: %w", src, err)
+	}
+	if err := out.Chmod(promotedFileMode); err != nil {
+		return fmt.Errorf("set destination file permissions %q: %w", dst, err)
+	}
+	if err := out.Sync(); err != nil {
+		return fmt.Errorf("sync destination file %q: %w", dst, err)
+	}
+	if err := out.Close(); err != nil {
+		return fmt.Errorf("close destination file %q: %w", dst, err)
+	}
+	closed = true
+	return nil
+}
+
+func syncDirectory(path string) error {
+	directory, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = directory.Close() }()
+
+	err = directory.Sync()
+	if errors.Is(err, syscall.EINVAL) || errors.Is(err, syscall.ENOTSUP) {
+		return nil
+	}
+	return err
+}
+
+func pathWithin(parent, candidate string) (bool, error) {
+	absoluteParent, err := filepath.Abs(parent)
+	if err != nil {
+		return false, fmt.Errorf("resolve source directory: %w", err)
+	}
+	absoluteCandidate, err := filepath.Abs(candidate)
+	if err != nil {
+		return false, fmt.Errorf("resolve destination directory: %w", err)
+	}
+	relative, err := filepath.Rel(absoluteParent, absoluteCandidate)
+	if err != nil {
+		return false, fmt.Errorf("compare source and destination directories: %w", err)
+	}
+	return relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)), nil
+}

internal/fileops/directory_special_unix_test.go

@@ -0,0 +1,35 @@
+//go:build !windows
+
+package fileops
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+)
+
+func TestPromoteDirectoryRejectsNamedPipe(t *testing.T) {
+	root := t.TempDir()
+	src := filepath.Join(root, "source")
+	dst := filepath.Join(root, "destination")
+	if err := os.Mkdir(src, 0o755); err != nil {
+		t.Fatalf("Mkdir(source) error = %v", err)
+	}
+	pipe := filepath.Join(src, "events.pipe")
+	if err := syscall.Mkfifo(pipe, 0o600); err != nil {
+		t.Skipf("Mkfifo() unavailable: %v", err)
+	}
+
+	if err := PromoteDirectory(src, dst); err == nil {
+		t.Fatal("PromoteDirectory() error = nil, want named pipe rejection")
+	}
+	info, err := os.Lstat(pipe)
+	if err != nil || info.Mode()&os.ModeNamedPipe == 0 {
+		t.Fatalf("source named pipe was not preserved: info=%v err=%v", info, err)
+	}
+	if _, err := os.Lstat(dst); !os.IsNotExist(err) {
+		t.Fatalf("Lstat(destination) error = %v, want not exist", err)
+	}
+	assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
+}

internal/fileops/directory_test.go

@@ -0,0 +1,246 @@
+package fileops
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestPromoteDirectoryCopiesNestedRegularTree(t *testing.T) {
+	root := t.TempDir()
+	src := filepath.Join(root, "source")
+	dst := filepath.Join(root, "promoted")
+	mustWriteFile(t, filepath.Join(src, "z-last.txt"), []byte("last"), 0o777)
+	mustWriteFile(t, filepath.Join(src, "nested", "binary.dat"), []byte{0, 1, 2, 0xff}, 0o600)
+	mustWriteFile(t, filepath.Join(src, "a-first.txt"), []byte("first"), 0o400)
+	if err := os.Mkdir(filepath.Join(src, "empty"), 0o700); err != nil {
+		t.Fatalf("Mkdir(empty) error = %v", err)
+	}
+
+	if err := PromoteDirectory(src, dst); err != nil {
+		t.Fatalf("PromoteDirectory() error = %v", err)
+	}
+
+	wantLayout := []string{".", "a-first.txt", "empty", "nested", "nested/binary.dat", "z-last.txt"}
+	if got := treeLayout(t, dst); !reflect.DeepEqual(got, wantLayout) {
+		t.Fatalf("promoted layout = %#v, want %#v", got, wantLayout)
+	}
+	assertFileBytes(t, filepath.Join(dst, "a-first.txt"), []byte("first"))
+	assertFileBytes(t, filepath.Join(dst, "nested", "binary.dat"), []byte{0, 1, 2, 0xff})
+	assertFileBytes(t, filepath.Join(dst, "z-last.txt"), []byte("last"))
+
+	for _, path := range []string{dst, filepath.Join(dst, "nested"), filepath.Join(dst, "empty")} {
+		info, err := os.Stat(path)
+		if err != nil {
+			t.Fatalf("Stat(%q) error = %v", path, err)
+		}
+		if got := info.Mode().Perm(); got != promotedDirectoryMode {
+			t.Fatalf("directory mode for %q = %o, want %o", path, got, promotedDirectoryMode)
+		}
+	}
+	for _, path := range []string{filepath.Join(dst, "a-first.txt"), filepath.Join(dst, "nested", "binary.dat"), filepath.Join(dst, "z-last.txt")} {
+		info, err := os.Stat(path)
+		if err != nil {
+			t.Fatalf("Stat(%q) error = %v", path, err)
+		}
+		if got := info.Mode().Perm(); got != promotedFileMode {
+			t.Fatalf("file mode for %q = %o, want %o", path, got, promotedFileMode)
+		}
+	}
+
+	assertFileBytes(t, filepath.Join(src, "nested", "binary.dat"), []byte{0, 1, 2, 0xff})
+	assertNoMatchingTempDirectories(t, root, ".promoted.tmp-")
+}
+
+func TestPromoteDirectoryRejectsInvalidPaths(t *testing.T) {
+	root := t.TempDir()
+	src := filepath.Join(root, "source")
+	if err := os.Mkdir(src, 0o755); err != nil {
+		t.Fatalf("Mkdir(source) error = %v", err)
+	}
+
+	tests := []struct {
+		name string
+		src  string
+		dst  string
+	}{
+		{name: "empty source", src: "", dst: filepath.Join(root, "out-a")},
+		{name: "empty destination", src: src, dst: " "},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := PromoteDirectory(test.src, test.dst); err == nil {
+				t.Fatal("PromoteDirectory() error = nil, want path validation failure")
+			}
+		})
+	}
+}
+
+func TestPromoteDirectoryRejectsExistingDestination(t *testing.T) {
+	root := t.TempDir()
+	src := filepath.Join(root, "source")
+	dst := filepath.Join(root, "destination")
+	mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("source"), 0o644)
+	mustWriteFile(t, filepath.Join(dst, "value.txt"), []byte("existing"), 0o644)
+
+	err := PromoteDirectory(src, dst)
+	if err == nil || !strings.Contains(err.Error(), "already exists") {
+		t.Fatalf("PromoteDirectory() error = %v, want existing destination error", err)
+	}
+	assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("existing"))
+	assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("source"))
+	assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
+}
+
+func TestPromoteDirectoryRejectsNonDirectorySource(t *testing.T) {
+	root := t.TempDir()
+	src := filepath.Join(root, "source.txt")
+	dst := filepath.Join(root, "destination")
+	mustWriteFile(t, src, []byte("source"), 0o644)
+
+	if err := PromoteDirectory(src, dst); err == nil {
+		t.Fatal("PromoteDirectory() error = nil, want non-directory source error")
+	}
+	assertFileBytes(t, src, []byte("source"))
+	if _, err := os.Lstat(dst); !os.IsNotExist(err) {
+		t.Fatalf("Lstat(destination) error = %v, want not exist", err)
+	}
+}
+
+func TestPromoteDirectoryRejectsSymlinksWithoutFollowingThem(t *testing.T) {
+	root := t.TempDir()
+	externalFile := filepath.Join(root, "external.txt")
+	externalDirectory := filepath.Join(root, "external-directory")
+	mustWriteFile(t, externalFile, []byte("outside"), 0o644)
+	mustWriteFile(t, filepath.Join(externalDirectory, "secret.txt"), []byte("secret"), 0o644)
+
+	tests := []struct {
+		name   string
+		target string
+		link   string
+	}{
+		{name: "file", target: externalFile, link: "file-link"},
+		{name: "directory", target: externalDirectory, link: "directory-link"},
+		{name: "escaping", target: filepath.Join("..", "external.txt"), link: "escaping-link"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			src := filepath.Join(root, "source-"+test.name)
+			dst := filepath.Join(root, "destination-"+test.name)
+			if err := os.Mkdir(src, 0o755); err != nil {
+				t.Fatalf("Mkdir(source) error = %v", err)
+			}
+			if err := os.Symlink(test.target, filepath.Join(src, test.link)); err != nil {
+				t.Skipf("Symlink() unavailable: %v", err)
+			}
+
+			if err := PromoteDirectory(src, dst); err == nil {
+				t.Fatal("PromoteDirectory() error = nil, want symlink rejection")
+			}
+			linkInfo, err := os.Lstat(filepath.Join(src, test.link))
+			if err != nil || linkInfo.Mode()&os.ModeSymlink == 0 {
+				t.Fatalf("source symlink was not preserved: info=%v err=%v", linkInfo, err)
+			}
+			if _, err := os.Lstat(dst); !os.IsNotExist(err) {
+				t.Fatalf("Lstat(destination) error = %v, want not exist", err)
+			}
+			assertNoMatchingTempDirectories(t, root, ".destination-"+test.name+".tmp-")
+		})
+	}
+	assertFileBytes(t, externalFile, []byte("outside"))
+	assertFileBytes(t, filepath.Join(externalDirectory, "secret.txt"), []byte("secret"))
+}
+
+func TestPromoteDirectoryDoesNotReplaceDestinationCreatedBeforeInstall(t *testing.T) {
+	root := t.TempDir()
+	src := filepath.Join(root, "source")
+	dst := filepath.Join(root, "destination")
+	mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("source"), 0o644)
+
+	err := promoteDirectory(src, dst, func(temporary, destination string) error {
+		if err := os.Mkdir(destination, 0o755); err != nil {
+			t.Fatalf("Mkdir(concurrent destination) error = %v", err)
+		}
+		mustWriteFile(t, filepath.Join(destination, "value.txt"), []byte("concurrent"), 0o644)
+		return renameDirectoryNoReplace(temporary, destination)
+	})
+	if err == nil {
+		t.Fatal("promoteDirectory() error = nil, want no-replace install failure")
+	}
+	assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("concurrent"))
+	assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("source"))
+	assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
+}
+
+func TestPromoteDirectoryRejectsDestinationInsideSource(t *testing.T) {
+	root := t.TempDir()
+	src := filepath.Join(root, "source")
+	dst := filepath.Join(src, "nested", "destination")
+	if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+		t.Fatalf("MkdirAll(destination parent) error = %v", err)
+	}
+
+	if err := PromoteDirectory(src, dst); err == nil {
+		t.Fatal("PromoteDirectory() error = nil, want nested destination rejection")
+	}
+	if _, err := os.Lstat(dst); !os.IsNotExist(err) {
+		t.Fatalf("Lstat(destination) error = %v, want not exist", err)
+	}
+}
+
+func mustWriteFile(t *testing.T, path string, data []byte, mode os.FileMode) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatalf("MkdirAll(%q) error = %v", filepath.Dir(path), err)
+	}
+	if err := os.WriteFile(path, data, mode); err != nil {
+		t.Fatalf("WriteFile(%q) error = %v", path, err)
+	}
+}
+
+func assertFileBytes(t *testing.T, path string, want []byte) {
+	t.Helper()
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("ReadFile(%q) error = %v", path, err)
+	}
+	if !bytes.Equal(got, want) {
+		t.Fatalf("ReadFile(%q) = %v, want %v", path, got, want)
+	}
+}
+
+func treeLayout(t *testing.T, root string) []string {
+	t.Helper()
+	var layout []string
+	err := filepath.WalkDir(root, func(path string, _ os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		relative, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		layout = append(layout, filepath.ToSlash(relative))
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("WalkDir(%q) error = %v", root, err)
+	}
+	return layout
+}
+
+func assertNoMatchingTempDirectories(t *testing.T, parent, prefix string) {
+	t.Helper()
+	entries, err := os.ReadDir(parent)
+	if err != nil {
+		t.Fatalf("ReadDir(%q) error = %v", parent, err)
+	}
+	for _, entry := range entries {
+		if strings.HasPrefix(entry.Name(), prefix) {
+			t.Fatalf("unexpected temporary directory residue: %s", filepath.Join(parent, entry.Name()))
+		}
+	}
+}

internal/fileops/rename_noreplace_linux.go

@@ -0,0 +1,9 @@
+//go:build linux
+
+package fileops
+
+import "golang.org/x/sys/unix"
+
+func renameDirectoryNoReplace(src, dst string) error {
+	return unix.Renameat2(unix.AT_FDCWD, src, unix.AT_FDCWD, dst, unix.RENAME_NOREPLACE)
+}

internal/fileops/rename_noreplace_other.go

@@ -0,0 +1,9 @@
+//go:build !linux
+
+package fileops
+
+import "fmt"
+
+func renameDirectoryNoReplace(_, _ string) error {
+	return fmt.Errorf("atomic no-replace directory rename is unsupported on this platform")
+}