Add storage backends and safety checks

2026-05-31 02:00:24 +00:00
parent 29dbad2967
commit 3c2f36a6e5
10 changed files with 1582 additions and 0 deletions
--- a/internal/storage/fake/backend.go
+++ b/internal/storage/fake/backend.go
@@ -0,0 +1,316 @@
+package fake
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"sort"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+const backendName = "fake"
+
+type Backend struct {
+	files    map[string][]byte
+	dirs     map[string]struct{}
+	symlinks map[string]struct{}
+}
+
+func New() *Backend {
+	return &Backend{
+		files:    make(map[string][]byte),
+		dirs:     map[string]struct{}{"": {}},
+		symlinks: make(map[string]struct{}),
+	}
+}
+
+func (b *Backend) AddDirectory(path string) error {
+	if err := storage.ValidatePrefix(path); err != nil {
+		return err
+	}
+	b.ensureParents(path)
+	b.dirs[path] = struct{}{}
+	return nil
+}
+
+func (b *Backend) AddSymlink(path string) error {
+	if err := storage.ValidatePath(path); err != nil {
+		return err
+	}
+	b.ensureParents(path)
+	delete(b.files, path)
+	delete(b.dirs, path)
+	b.symlinks[path] = struct{}{}
+	return nil
+}
+
+func (b *Backend) ReadFile(ctx context.Context, path string) ([]byte, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	if err := storage.ValidatePath(path); err != nil {
+		return nil, err
+	}
+	data, ok := b.files[path]
+	if !ok {
+		if b.exists(path) {
+			return nil, storage.NewError(storage.OpReadFile, backendName, path, storage.ErrUnsupported, nil)
+		}
+		return nil, storage.NewError(storage.OpReadFile, backendName, path, storage.ErrNotFound, nil)
+	}
+	return append([]byte(nil), data...), nil
+}
+
+func (b *Backend) OpenReader(ctx context.Context, path string) (io.ReadCloser, error) {
+	data, err := b.ReadFile(ctx, path)
+	if err != nil {
+		return nil, err
+	}
+	return io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (b *Backend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
+	opts.Size = int64(len(data))
+	opts.SizeKnown = true
+	return b.WriteFrom(ctx, path, bytes.NewReader(data), opts)
+}
+
+func (b *Backend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	if err := storage.ValidatePath(path); err != nil {
+		return storage.Entry{}, err
+	}
+	if b.exists(path) && !opts.Overwrite {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrAlreadyExist, nil)
+	}
+	if _, ok := b.dirs[path]; ok {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, nil)
+	}
+	if _, ok := b.symlinks[path]; ok {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, nil)
+	}
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrUnknown, err)
+	}
+	if opts.SizeKnown && int64(len(data)) != opts.Size {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, nil)
+	}
+	b.ensureParents(path)
+	b.files[path] = append([]byte(nil), data...)
+	delete(b.symlinks, path)
+	return storage.Entry{Path: path, Type: storage.EntryTypeFile, Size: int64(len(data))}, nil
+}
+
+func (b *Backend) Stat(ctx context.Context, path string) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	if err := storage.ValidatePrefix(path); err != nil {
+		return storage.Entry{}, err
+	}
+	if data, ok := b.files[path]; ok {
+		return storage.Entry{Path: path, Type: storage.EntryTypeFile, Size: int64(len(data))}, nil
+	}
+	if _, ok := b.symlinks[path]; ok {
+		return storage.Entry{Path: path, Type: storage.EntryTypeSymlink}, nil
+	}
+	if _, ok := b.dirs[path]; ok {
+		return storage.Entry{Path: path, Type: storage.EntryTypeDirectory}, nil
+	}
+	return storage.Entry{}, storage.NewError(storage.OpStat, backendName, path, storage.ErrNotFound, nil)
+}
+
+func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if err := storage.ValidatePrefix(prefix); err != nil {
+		return err
+	}
+	if entry, err := b.Stat(ctx, prefix); err == nil && entry.Type != storage.EntryTypeDirectory {
+		return emit(ctx, entry, opts, fn)
+	} else if err != nil && !storage.IsNotFound(err) {
+		return err
+	}
+
+	entries := b.entries()
+	visited := 0
+	for _, entry := range entries {
+		if entry.Path == "" || !entryBelow(prefix, entry.Path) {
+			continue
+		}
+		if !opts.Recursive && !isImmediateChild(prefix, entry.Path) {
+			continue
+		}
+		if opts.Limit > 0 && visited >= opts.Limit {
+			return nil
+		}
+		visited++
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if err := fn(entry); err != nil {
+			if errors.Is(err, storage.ErrStopWalk) {
+				return nil
+			}
+			return storage.NewError(storage.OpWalk, backendName, entry.Path, storage.ErrUnknown, err)
+		}
+	}
+	return nil
+}
+
+func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
+	found := false
+	err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
+		found = true
+		return storage.ErrStopWalk
+	})
+	if err != nil {
+		return false, err
+	}
+	return found, nil
+}
+
+func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if err := storage.ValidatePrefix(bundlePath); err != nil {
+		return err
+	}
+	targets := make([]string, 0, len(managedOutputPaths)+1)
+	for _, outputPath := range managedOutputPaths {
+		target, err := storage.Join(bundlePath, outputPath)
+		if err != nil {
+			return err
+		}
+		targets = append(targets, target)
+	}
+	statePath, err := storage.StatePath(bundlePath)
+	if err != nil {
+		return err
+	}
+	targets = append(targets, statePath)
+
+	for _, target := range targets {
+		if _, ok := b.dirs[target]; ok {
+			return storage.NewError(storage.OpDeleteManagedBundle, backendName, target, storage.ErrUnsupported, nil)
+		}
+		if !b.exists(target) {
+			if opts.IgnoreMissing {
+				continue
+			}
+			return storage.NewError(storage.OpDeleteManagedBundle, backendName, target, storage.ErrNotFound, nil)
+		}
+		delete(b.files, target)
+		delete(b.symlinks, target)
+		if opts.PruneEmptyDirs {
+			b.pruneEmptyParents(parentOf(target))
+		}
+	}
+	return nil
+}
+
+func (b *Backend) ensureParents(path string) {
+	parent := parentOf(path)
+	for parent != "" {
+		b.dirs[parent] = struct{}{}
+		parent = parentOf(parent)
+	}
+	b.dirs[""] = struct{}{}
+}
+
+func (b *Backend) pruneEmptyParents(path string) {
+	for path != "" {
+		if b.hasChild(path) {
+			return
+		}
+		delete(b.dirs, path)
+		path = parentOf(path)
+	}
+}
+
+func (b *Backend) hasChild(path string) bool {
+	for candidate := range b.files {
+		if entryBelow(path, candidate) {
+			return true
+		}
+	}
+	for candidate := range b.symlinks {
+		if entryBelow(path, candidate) {
+			return true
+		}
+	}
+	for candidate := range b.dirs {
+		if candidate != path && entryBelow(path, candidate) {
+			return true
+		}
+	}
+	return false
+}
+
+func (b *Backend) exists(path string) bool {
+	_, file := b.files[path]
+	_, dir := b.dirs[path]
+	_, symlink := b.symlinks[path]
+	return file || dir || symlink
+}
+
+func (b *Backend) entries() []storage.Entry {
+	entries := make([]storage.Entry, 0, len(b.files)+len(b.dirs)+len(b.symlinks))
+	for path, data := range b.files {
+		entries = append(entries, storage.Entry{Path: path, Type: storage.EntryTypeFile, Size: int64(len(data))})
+	}
+	for path := range b.dirs {
+		entries = append(entries, storage.Entry{Path: path, Type: storage.EntryTypeDirectory})
+	}
+	for path := range b.symlinks {
+		entries = append(entries, storage.Entry{Path: path, Type: storage.EntryTypeSymlink})
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].Path < entries[j].Path
+	})
+	return entries
+}
+
+func emit(ctx context.Context, entry storage.Entry, opts storage.WalkOptions, fn storage.WalkFunc) error {
+	if opts.Limit > 0 && opts.Limit < 1 {
+		return nil
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if err := fn(entry); err != nil && !errors.Is(err, storage.ErrStopWalk) {
+		return err
+	}
+	return nil
+}
+
+func entryBelow(prefix, path string) bool {
+	if prefix == "" {
+		return path != ""
+	}
+	return strings.HasPrefix(path, prefix+"/")
+}
+
+func isImmediateChild(prefix, path string) bool {
+	remainder := path
+	if prefix != "" {
+		remainder = strings.TrimPrefix(path, prefix+"/")
+	}
+	return !strings.Contains(remainder, "/")
+}
+
+func parentOf(path string) string {
+	index := strings.LastIndex(path, "/")
+	if index == -1 {
+		return ""
+	}
+	return path[:index]
+}
--- a/internal/storage/fake/backend_test.go
+++ b/internal/storage/fake/backend_test.go
@@ -0,0 +1,187 @@
+package fake
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"reflect"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+func TestBackendReadWriteAndStream(t *testing.T) {
+	backend := New()
+	entry, err := backend.WriteFile(context.Background(), "reports/report.md", []byte("hello"), storage.WriteOptions{})
+	if err != nil {
+		t.Fatalf("WriteFile() error = %v", err)
+	}
+	if entry.Path != "reports/report.md" || entry.Type != storage.EntryTypeFile || entry.Size != 5 {
+		t.Fatalf("entry = %#v, want file metadata", entry)
+	}
+
+	data, err := backend.ReadFile(context.Background(), "reports/report.md")
+	if err != nil {
+		t.Fatalf("ReadFile() error = %v", err)
+	}
+	reader, err := backend.OpenReader(context.Background(), "reports/report.md")
+	if err != nil {
+		t.Fatalf("OpenReader() error = %v", err)
+	}
+	streamed, err := io.ReadAll(reader)
+	closeErr := reader.Close()
+	if err != nil || closeErr != nil {
+		t.Fatalf("stream read error = %v close = %v", err, closeErr)
+	}
+	if !bytes.Equal(data, streamed) {
+		t.Fatalf("streamed = %q, want %q", streamed, data)
+	}
+
+	_, err = backend.WriteFile(context.Background(), "reports/report.md", []byte("again"), storage.WriteOptions{})
+	if !storage.IsAlreadyExists(err) {
+		t.Fatalf("WriteFile existing error = %v, want already exists", err)
+	}
+}
+
+func TestBackendStatWalkAndList(t *testing.T) {
+	backend := New()
+	mustWrite(t, backend, "b/two.txt", "2")
+	mustWrite(t, backend, "a/one.txt", "1")
+
+	entry, err := backend.Stat(context.Background(), "a")
+	if err != nil {
+		t.Fatalf("Stat directory error = %v", err)
+	}
+	if entry.Type != storage.EntryTypeDirectory {
+		t.Fatalf("entry type = %s, want directory", entry.Type)
+	}
+
+	entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: true})
+	if err != nil {
+		t.Fatalf("List() error = %v", err)
+	}
+	var paths []string
+	for _, entry := range entries {
+		paths = append(paths, entry.Path)
+	}
+	want := []string{"a", "a/one.txt", "b", "b/two.txt"}
+	if !reflect.DeepEqual(paths, want) {
+		t.Fatalf("paths = %v, want %v", paths, want)
+	}
+}
+
+func TestBackendRejectsInvalidPaths(t *testing.T) {
+	backend := New()
+	_, err := backend.WriteFile(context.Background(), "../outside", []byte("data"), storage.WriteOptions{})
+	if !storage.IsInvalidPath(err) {
+		t.Fatalf("WriteFile traversal error = %v, want invalid path", err)
+	}
+	_, err = backend.ReadFile(context.Background(), `bad\path`)
+	if !storage.IsInvalidPath(err) {
+		t.Fatalf("ReadFile backslash error = %v, want invalid path", err)
+	}
+}
+
+func TestBackendSymlinkReportingAndReadRejection(t *testing.T) {
+	backend := New()
+	if err := backend.AddSymlink("link.txt"); err != nil {
+		t.Fatalf("AddSymlink() error = %v", err)
+	}
+	entry, err := backend.Stat(context.Background(), "link.txt")
+	if err != nil {
+		t.Fatalf("Stat() error = %v", err)
+	}
+	if entry.Type != storage.EntryTypeSymlink {
+		t.Fatalf("entry type = %s, want symlink", entry.Type)
+	}
+	_, err = backend.ReadFile(context.Background(), "link.txt")
+	if !storage.IsUnsupported(err) {
+		t.Fatalf("ReadFile symlink error = %v, want unsupported", err)
+	}
+}
+
+func TestBackendWriteFromSizeMismatch(t *testing.T) {
+	backend := New()
+	_, err := backend.WriteFrom(context.Background(), "out.txt", bytes.NewBufferString("short"), storage.WriteOptions{SizeKnown: true, Size: 99})
+	if !storage.IsConflict(err) {
+		t.Fatalf("WriteFrom size mismatch error = %v, want conflict", err)
+	}
+	if _, err := backend.Stat(context.Background(), "out.txt"); !storage.IsNotFound(err) {
+		t.Fatalf("Stat after failed write error = %v, want not found", err)
+	}
+}
+
+func TestBackendManagedDeletion(t *testing.T) {
+	backend := New()
+	mustWrite(t, backend, "bundle/report.html", "html")
+	mustWrite(t, backend, "bundle/keep.txt", "keep")
+	mustWrite(t, backend, "bundle/.distributor.json", "{}")
+
+	err := backend.DeleteManagedBundle(context.Background(), "bundle", []string{"report.html"}, storage.DeleteOptions{PruneEmptyDirs: true})
+	if err != nil {
+		t.Fatalf("DeleteManagedBundle() error = %v", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle/report.html"); !storage.IsNotFound(err) {
+		t.Fatalf("managed output stat error = %v, want not found", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle/.distributor.json"); !storage.IsNotFound(err) {
+		t.Fatalf("state stat error = %v, want not found", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle/keep.txt"); err != nil {
+		t.Fatalf("unlisted file stat error = %v", err)
+	}
+}
+
+func TestBackendHasAnyAndWalkStop(t *testing.T) {
+	backend := New()
+	found, err := backend.HasAny(context.Background(), "missing")
+	if err != nil {
+		t.Fatalf("HasAny missing error = %v", err)
+	}
+	if found {
+		t.Fatal("HasAny missing = true, want false")
+	}
+	mustWrite(t, backend, "bundle/report.md", "report")
+	found, err = backend.HasAny(context.Background(), "bundle")
+	if err != nil {
+		t.Fatalf("HasAny bundle error = %v", err)
+	}
+	if !found {
+		t.Fatal("HasAny bundle = false, want true")
+	}
+	found, err = backend.HasAny(context.Background(), "bund")
+	if err != nil {
+		t.Fatalf("HasAny sibling prefix error = %v", err)
+	}
+	if found {
+		t.Fatal("HasAny sibling prefix = true, want false")
+	}
+
+	visited := 0
+	err = backend.Walk(context.Background(), "", storage.WalkOptions{Recursive: true}, func(storage.Entry) error {
+		visited++
+		return storage.ErrStopWalk
+	})
+	if err != nil {
+		t.Fatalf("Walk stop error = %v", err)
+	}
+	if visited != 1 {
+		t.Fatalf("visited = %d, want 1", visited)
+	}
+
+	errSentinel := errors.New("callback")
+	err = backend.Walk(context.Background(), "", storage.WalkOptions{Recursive: true}, func(storage.Entry) error {
+		return errSentinel
+	})
+	if !errors.Is(err, errSentinel) {
+		t.Fatalf("Walk callback error = %v, want sentinel", err)
+	}
+}
+
+func mustWrite(t *testing.T, backend *Backend, path, data string) {
+	t.Helper()
+	if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
+		t.Fatalf("WriteFile(%q) error = %v", path, err)
+	}
+}