Add storage backends and safety checks

internal/adapters/local/backend.go

@@ -0,0 +1,370 @@
+package local
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+const backendName = "local"
+
+type Backend struct {
+	root string
+}
+
+func New(root string) (*Backend, error) {
+	if root == "" {
+		return nil, storage.NewError(storage.OpOpenBackend, backendName, "", storage.ErrInvalidPath, nil)
+	}
+	absRoot, err := filepath.Abs(root)
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, backendName, root, storage.ErrInvalidPath, err)
+	}
+	return &Backend{root: filepath.Clean(absRoot)}, nil
+}
+
+func (b *Backend) ReadFile(ctx context.Context, path string) ([]byte, error) {
+	reader, err := b.OpenReader(ctx, path)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, storage.NewError(storage.OpReadFile, backendName, path, storage.ErrUnknown, err)
+	}
+	return data, nil
+}
+
+func (b *Backend) OpenReader(ctx context.Context, path string) (io.ReadCloser, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	nativePath, err := b.nativePath(path, false)
+	if err != nil {
+		return nil, err
+	}
+	if err := b.rejectSymlinkAncestors(nativePath, true); err != nil {
+		return nil, err
+	}
+	info, err := os.Lstat(nativePath)
+	if err != nil {
+		return nil, b.translateError(storage.OpOpenReader, path, err)
+	}
+	if !info.Mode().IsRegular() {
+		return nil, storage.NewError(storage.OpOpenReader, backendName, path, storage.ErrUnsupported, nil)
+	}
+	file, err := os.Open(nativePath)
+	if err != nil {
+		return nil, b.translateError(storage.OpOpenReader, path, err)
+	}
+	return file, nil
+}
+
+func (b *Backend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
+	opts.Size = int64(len(data))
+	opts.SizeKnown = true
+	return b.WriteFrom(ctx, path, bytes.NewReader(data), opts)
+}
+
+func (b *Backend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	nativePath, err := b.nativePath(path, false)
+	if err != nil {
+		return storage.Entry{}, err
+	}
+	parent := filepath.Dir(nativePath)
+	if err := b.rejectSymlinkAncestors(parent, true); err != nil {
+		return storage.Entry{}, err
+	}
+	if info, err := os.Lstat(nativePath); err == nil {
+		if !opts.Overwrite {
+			return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrAlreadyExist, nil)
+		}
+		if !info.Mode().IsRegular() {
+			return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, nil)
+		}
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, path, err)
+	}
+
+	if err := os.MkdirAll(parent, 0o755); err != nil {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, path, err)
+	}
+	temp, err := os.CreateTemp(parent, ".distributor-write-*")
+	if err != nil {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, path, err)
+	}
+	tempPath := temp.Name()
+	cleanup := true
+	defer func() {
+		if cleanup {
+			_ = os.Remove(tempPath)
+		}
+	}()
+
+	written, copyErr := io.Copy(temp, r)
+	closeErr := temp.Close()
+	if copyErr != nil {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrUnknown, copyErr)
+	}
+	if closeErr != nil {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrUnknown, closeErr)
+	}
+	if opts.SizeKnown && written != opts.Size {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", written, opts.Size))
+	}
+	if err := os.Rename(tempPath, nativePath); err != nil {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, path, err)
+	}
+	cleanup = false
+	return b.Stat(ctx, path)
+}
+
+func (b *Backend) Stat(ctx context.Context, path string) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	nativePath, err := b.nativePath(path, true)
+	if err != nil {
+		return storage.Entry{}, err
+	}
+	info, err := os.Lstat(nativePath)
+	if err != nil {
+		return storage.Entry{}, b.translateError(storage.OpStat, path, err)
+	}
+	return entryFromInfo(path, info), nil
+}
+
+func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	nativePrefix, err := b.nativePath(prefix, true)
+	if err != nil {
+		return err
+	}
+	info, err := os.Lstat(nativePrefix)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil
+		}
+		return b.translateError(storage.OpWalk, prefix, err)
+	}
+	visited := 0
+	emit := func(entry storage.Entry) error {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if opts.Limit > 0 && visited >= opts.Limit {
+			return storage.ErrStopWalk
+		}
+		visited++
+		if err := fn(entry); err != nil {
+			if errors.Is(err, storage.ErrStopWalk) {
+				return storage.ErrStopWalk
+			}
+			return storage.NewError(storage.OpWalk, backendName, entry.Path, storage.ErrUnknown, err)
+		}
+		return nil
+	}
+
+	if !info.IsDir() {
+		if err := emit(entryFromInfo(prefix, info)); errors.Is(err, storage.ErrStopWalk) {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	walkErr := filepath.WalkDir(nativePrefix, func(nativePath string, dirEntry fs.DirEntry, err error) error {
+		if err != nil {
+			return b.translateError(storage.OpWalk, prefix, err)
+		}
+		if nativePath == nativePrefix {
+			return nil
+		}
+		relPath, err := filepath.Rel(b.root, nativePath)
+		if err != nil {
+			return b.translateError(storage.OpWalk, prefix, err)
+		}
+		logicalPath := filepath.ToSlash(relPath)
+		if !opts.Recursive && filepath.Dir(nativePath) != nativePrefix {
+			if dirEntry.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		info, err := dirEntry.Info()
+		if err != nil {
+			return b.translateError(storage.OpWalk, logicalPath, err)
+		}
+		return emit(entryFromInfo(logicalPath, info))
+	})
+	if errors.Is(walkErr, storage.ErrStopWalk) {
+		return nil
+	}
+	return walkErr
+}
+
+func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
+	found := false
+	err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
+		found = true
+		return storage.ErrStopWalk
+	})
+	if err != nil {
+		return false, err
+	}
+	return found, nil
+}
+
+func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if err := storage.ValidatePrefix(bundlePath); err != nil {
+		return err
+	}
+	targets := make([]string, 0, len(managedOutputPaths)+1)
+	for _, outputPath := range managedOutputPaths {
+		target, err := storage.Join(bundlePath, outputPath)
+		if err != nil {
+			return err
+		}
+		targets = append(targets, target)
+	}
+	statePath, err := storage.StatePath(bundlePath)
+	if err != nil {
+		return err
+	}
+	targets = append(targets, statePath)
+
+	for _, logicalPath := range targets {
+		nativePath, err := b.nativePath(logicalPath, false)
+		if err != nil {
+			return err
+		}
+		if nativePath == b.root {
+			return storage.NewError(storage.OpDeleteManagedBundle, backendName, logicalPath, storage.ErrInvalidPath, nil)
+		}
+		info, err := os.Lstat(nativePath)
+		if err != nil {
+			if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
+				continue
+			}
+			return b.translateError(storage.OpDeleteManagedBundle, logicalPath, err)
+		}
+		if info.IsDir() {
+			return storage.NewError(storage.OpDeleteManagedBundle, backendName, logicalPath, storage.ErrUnsupported, nil)
+		}
+		if err := os.Remove(nativePath); err != nil {
+			return b.translateError(storage.OpDeleteManagedBundle, logicalPath, err)
+		}
+		if opts.PruneEmptyDirs {
+			b.pruneEmptyParents(filepath.Dir(nativePath))
+		}
+	}
+	return nil
+}
+
+func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) {
+	if logicalPath == "" {
+		if !allowEmpty {
+			return "", storage.NewError(storage.OpValidatePath, backendName, logicalPath, storage.ErrInvalidPath, nil)
+		}
+		return b.root, nil
+	}
+	if err := storage.ValidatePath(logicalPath); err != nil {
+		return "", err
+	}
+	nativePath := filepath.Clean(filepath.Join(b.root, filepath.FromSlash(logicalPath)))
+	rel, err := filepath.Rel(b.root, nativePath)
+	if err != nil {
+		return "", storage.NewError(storage.OpValidatePath, backendName, logicalPath, storage.ErrInvalidPath, err)
+	}
+	if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
+		return "", storage.NewError(storage.OpValidatePath, backendName, logicalPath, storage.ErrInvalidPath, nil)
+	}
+	return nativePath, nil
+}
+
+func (b *Backend) rejectSymlinkAncestors(nativePath string, includeFinal bool) error {
+	rel, err := filepath.Rel(b.root, nativePath)
+	if err != nil {
+		return storage.NewError(storage.OpValidatePath, backendName, nativePath, storage.ErrInvalidPath, err)
+	}
+	if rel == "." {
+		return nil
+	}
+	segments := strings.Split(rel, string(filepath.Separator))
+	limit := len(segments)
+	if !includeFinal {
+		limit--
+	}
+	current := b.root
+	for i := 0; i < limit; i++ {
+		current = filepath.Join(current, segments[i])
+		info, err := os.Lstat(current)
+		if err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				return nil
+			}
+			return b.translateError(storage.OpStat, filepath.ToSlash(filepath.Join(segments[:i+1]...)), err)
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			return storage.NewError(storage.OpStat, backendName, filepath.ToSlash(filepath.Join(segments[:i+1]...)), storage.ErrUnsupported, nil)
+		}
+	}
+	return nil
+}
+
+func (b *Backend) pruneEmptyParents(start string) {
+	for current := start; current != b.root && strings.HasPrefix(current, b.root); current = filepath.Dir(current) {
+		err := os.Remove(current)
+		if err != nil {
+			return
+		}
+	}
+}
+
+func (b *Backend) translateError(op, path string, err error) error {
+	kind := storage.ErrUnknown
+	switch {
+	case errors.Is(err, fs.ErrNotExist):
+		kind = storage.ErrNotFound
+	case errors.Is(err, fs.ErrExist):
+		kind = storage.ErrAlreadyExist
+	case errors.Is(err, fs.ErrPermission):
+		kind = storage.ErrPermission
+	}
+	return storage.NewError(op, backendName, path, kind, err)
+}
+
+func entryFromInfo(path string, info fs.FileInfo) storage.Entry {
+	entryType := storage.EntryTypeOther
+	switch {
+	case info.Mode()&os.ModeSymlink != 0:
+		entryType = storage.EntryTypeSymlink
+	case info.Mode().IsRegular():
+		entryType = storage.EntryTypeFile
+	case info.IsDir():
+		entryType = storage.EntryTypeDirectory
+	}
+	return storage.Entry{
+		Path: path,
+		Type: entryType,
+		Size: info.Size(),
+	}
+}

internal/adapters/local/backend_test.go

@@ -0,0 +1,233 @@
+package local
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+func TestBackendRejectsTraversal(t *testing.T) {
+	backend := newBackend(t)
+	_, err := backend.ReadFile(context.Background(), "../outside")
+	if !storage.IsInvalidPath(err) {
+		t.Fatalf("ReadFile traversal error = %v, want invalid path", err)
+	}
+	_, err = backend.WriteFile(context.Background(), "/absolute", []byte("data"), storage.WriteOptions{})
+	if !storage.IsInvalidPath(err) {
+		t.Fatalf("WriteFile absolute path error = %v, want invalid path", err)
+	}
+}
+
+func TestBackendReadWriteAndStream(t *testing.T) {
+	backend := newBackend(t)
+
+	entry, err := backend.WriteFile(context.Background(), "reports/report.md", []byte("hello"), storage.WriteOptions{})
+	if err != nil {
+		t.Fatalf("WriteFile() error = %v", err)
+	}
+	if entry.Path != "reports/report.md" || entry.Type != storage.EntryTypeFile || entry.Size != 5 {
+		t.Fatalf("entry = %#v, want written file metadata", entry)
+	}
+
+	data, err := backend.ReadFile(context.Background(), "reports/report.md")
+	if err != nil {
+		t.Fatalf("ReadFile() error = %v", err)
+	}
+	if string(data) != "hello" {
+		t.Fatalf("ReadFile() = %q, want hello", data)
+	}
+
+	reader, err := backend.OpenReader(context.Background(), "reports/report.md")
+	if err != nil {
+		t.Fatalf("OpenReader() error = %v", err)
+	}
+	streamed, err := io.ReadAll(reader)
+	closeErr := reader.Close()
+	if err != nil || closeErr != nil {
+		t.Fatalf("read stream error = %v close = %v", err, closeErr)
+	}
+	if !bytes.Equal(streamed, data) {
+		t.Fatalf("streamed = %q, want %q", streamed, data)
+	}
+
+	_, err = backend.WriteFile(context.Background(), "reports/report.md", []byte("again"), storage.WriteOptions{})
+	if !storage.IsAlreadyExists(err) {
+		t.Fatalf("WriteFile without overwrite error = %v, want already exists", err)
+	}
+}
+
+func TestBackendStatWalkAndList(t *testing.T) {
+	backend := newBackend(t)
+	mustWrite(t, backend, "b/two.txt", "2")
+	mustWrite(t, backend, "a/one.txt", "1")
+
+	entry, err := backend.Stat(context.Background(), "a/one.txt")
+	if err != nil {
+		t.Fatalf("Stat() error = %v", err)
+	}
+	if entry.Type != storage.EntryTypeFile || entry.Size != 1 {
+		t.Fatalf("entry = %#v, want file size 1", entry)
+	}
+
+	entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: true})
+	if err != nil {
+		t.Fatalf("List() error = %v", err)
+	}
+	var paths []string
+	for _, entry := range entries {
+		paths = append(paths, entry.Path)
+	}
+	want := []string{"a", "a/one.txt", "b", "b/two.txt"}
+	if !reflect.DeepEqual(paths, want) {
+		t.Fatalf("paths = %v, want %v", paths, want)
+	}
+
+	entries, err = storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: false})
+	if err != nil {
+		t.Fatalf("List nonrecursive error = %v", err)
+	}
+	paths = paths[:0]
+	for _, entry := range entries {
+		paths = append(paths, entry.Path)
+	}
+	want = []string{"a", "b"}
+	if !reflect.DeepEqual(paths, want) {
+		t.Fatalf("nonrecursive paths = %v, want %v", paths, want)
+	}
+}
+
+func TestBackendSymlinkReportingAndReadRejection(t *testing.T) {
+	root := t.TempDir()
+	backend, err := New(root)
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "target.txt"), []byte("target"), 0o600); err != nil {
+		t.Fatalf("write target: %v", err)
+	}
+	if err := os.Symlink("target.txt", filepath.Join(root, "link.txt")); err != nil {
+		t.Fatalf("symlink: %v", err)
+	}
+
+	entry, err := backend.Stat(context.Background(), "link.txt")
+	if err != nil {
+		t.Fatalf("Stat() error = %v", err)
+	}
+	if entry.Type != storage.EntryTypeSymlink {
+		t.Fatalf("entry type = %s, want symlink", entry.Type)
+	}
+	_, err = backend.ReadFile(context.Background(), "link.txt")
+	if !storage.IsUnsupported(err) {
+		t.Fatalf("ReadFile symlink error = %v, want unsupported", err)
+	}
+}
+
+func TestBackendWriteFromSizeMismatchLeavesNoFinalFile(t *testing.T) {
+	backend := newBackend(t)
+	_, err := backend.WriteFrom(context.Background(), "out.txt", bytes.NewBufferString("short"), storage.WriteOptions{SizeKnown: true, Size: 99})
+	if !storage.IsConflict(err) {
+		t.Fatalf("WriteFrom size mismatch error = %v, want conflict", err)
+	}
+	_, err = backend.Stat(context.Background(), "out.txt")
+	if !storage.IsNotFound(err) {
+		t.Fatalf("Stat after failed write error = %v, want not found", err)
+	}
+}
+
+func TestBackendManagedDeletion(t *testing.T) {
+	backend := newBackend(t)
+	mustWrite(t, backend, "bundle/report.html", "html")
+	mustWrite(t, backend, "bundle/keep.txt", "keep")
+	mustWrite(t, backend, "bundle/.distributor.json", "{}")
+
+	err := backend.DeleteManagedBundle(context.Background(), "bundle", []string{"report.html"}, storage.DeleteOptions{PruneEmptyDirs: true})
+	if err != nil {
+		t.Fatalf("DeleteManagedBundle() error = %v", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle/report.html"); !storage.IsNotFound(err) {
+		t.Fatalf("managed output stat error = %v, want not found", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle/.distributor.json"); !storage.IsNotFound(err) {
+		t.Fatalf("state stat error = %v, want not found", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle/keep.txt"); err != nil {
+		t.Fatalf("unlisted file stat error = %v", err)
+	}
+	if err := backend.DeleteManagedBundle(context.Background(), "bundle", []string{""}, storage.DeleteOptions{}); !storage.IsInvalidPath(err) {
+		t.Fatalf("DeleteManagedBundle invalid output error = %v, want invalid path", err)
+	}
+}
+
+func TestBackendHasAny(t *testing.T) {
+	backend := newBackend(t)
+	found, err := backend.HasAny(context.Background(), "missing")
+	if err != nil {
+		t.Fatalf("HasAny missing error = %v", err)
+	}
+	if found {
+		t.Fatal("HasAny missing = true, want false")
+	}
+	mustWrite(t, backend, "bundle/report.md", "report")
+	found, err = backend.HasAny(context.Background(), "bundle")
+	if err != nil {
+		t.Fatalf("HasAny bundle error = %v", err)
+	}
+	if !found {
+		t.Fatal("HasAny bundle = false, want true")
+	}
+	found, err = backend.HasAny(context.Background(), "bund")
+	if err != nil {
+		t.Fatalf("HasAny sibling prefix error = %v", err)
+	}
+	if found {
+		t.Fatal("HasAny prefix sibling = true, want false")
+	}
+}
+
+func TestBackendWalkStops(t *testing.T) {
+	backend := newBackend(t)
+	mustWrite(t, backend, "a.txt", "a")
+	mustWrite(t, backend, "b.txt", "b")
+	visited := 0
+	err := backend.Walk(context.Background(), "", storage.WalkOptions{Recursive: true}, func(storage.Entry) error {
+		visited++
+		return storage.ErrStopWalk
+	})
+	if err != nil {
+		t.Fatalf("Walk() error = %v", err)
+	}
+	if visited != 1 {
+		t.Fatalf("visited = %d, want 1", visited)
+	}
+
+	errSentinel := errors.New("callback")
+	err = backend.Walk(context.Background(), "", storage.WalkOptions{Recursive: true}, func(storage.Entry) error {
+		return errSentinel
+	})
+	if !errors.Is(err, errSentinel) {
+		t.Fatalf("Walk callback error = %v, want sentinel", err)
+	}
+}
+
+func newBackend(t *testing.T) *Backend {
+	t.Helper()
+	backend, err := New(t.TempDir())
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	return backend
+}
+
+func mustWrite(t *testing.T, backend *Backend, path, data string) {
+	t.Helper()
+	if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
+		t.Fatalf("WriteFile(%q) error = %v", path, err)
+	}
+}