Add S3-compatible storage backend

2026-05-31 17:11:53 +00:00
parent 052aa8a64a
commit 14fa9c8000
27 changed files with 1334 additions and 45 deletions
--- a/internal/adapters/s3/backend.go
+++ b/internal/adapters/s3/backend.go
@@ -0,0 +1,430 @@
+package s3
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"sort"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awscfg "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+)
+
+type Client interface {
+	HeadObject(ctx context.Context, input *awss3.HeadObjectInput, optFns ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error)
+	GetObject(ctx context.Context, input *awss3.GetObjectInput, optFns ...func(*awss3.Options)) (*awss3.GetObjectOutput, error)
+	PutObject(ctx context.Context, input *awss3.PutObjectInput, optFns ...func(*awss3.Options)) (*awss3.PutObjectOutput, error)
+	ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, optFns ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error)
+	DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, optFns ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error)
+}
+
+type Backend struct {
+	client Client
+	bucket string
+	prefix string
+}
+
+func New(ctx context.Context, options Options) (*Backend, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	options, err := options.normalized()
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Prefix, storage.ErrInvalidPath, err)
+	}
+	loadOptions := []func(*awscfg.LoadOptions) error{
+		awscfg.WithRegion(options.Region),
+	}
+	if options.AccessKeyID != "" {
+		loadOptions = append(loadOptions, awscfg.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(options.AccessKeyID, options.SecretAccessKey, "")))
+	}
+	cfg, err := awscfg.LoadDefaultConfig(ctx, loadOptions...)
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Endpoint, storage.ErrUnknown, err)
+	}
+	client := awss3.NewFromConfig(cfg, func(o *awss3.Options) {
+		o.BaseEndpoint = aws.String(options.Endpoint)
+		o.UsePathStyle = options.ForcePathStyle
+	})
+	return NewWithClient(client, options)
+}
+
+func NewWithClient(client Client, options Options) (*Backend, error) {
+	options, err := options.normalized()
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Prefix, storage.ErrInvalidPath, err)
+	}
+	if client == nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Bucket, storage.ErrInvalidPath, fmt.Errorf("client is required"))
+	}
+	return &Backend{
+		client: client,
+		bucket: options.Bucket,
+		prefix: options.Prefix,
+	}, nil
+}
+
+func (b *Backend) ReadFile(ctx context.Context, logicalPath string) ([]byte, error) {
+	reader, err := b.OpenReader(ctx, logicalPath)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, storage.NewError(storage.OpReadFile, BackendName, logicalPath, storage.ErrUnknown, err)
+	}
+	return data, nil
+}
+
+func (b *Backend) OpenReader(ctx context.Context, logicalPath string) (io.ReadCloser, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	key, err := b.objectKey(logicalPath, false)
+	if err != nil {
+		return nil, err
+	}
+	output, err := b.client.GetObject(ctx, &awss3.GetObjectInput{
+		Bucket: aws.String(b.bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
+	}
+	return output.Body, nil
+}
+
+func (b *Backend) WriteFile(ctx context.Context, logicalPath string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
+	opts.Size = int64(len(data))
+	opts.SizeKnown = true
+	return b.WriteFrom(ctx, logicalPath, bytes.NewReader(data), opts)
+}
+
+func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	key, err := b.objectKey(logicalPath, false)
+	if err != nil {
+		return storage.Entry{}, err
+	}
+	if !opts.Overwrite {
+		_, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
+			Bucket: aws.String(b.bucket),
+			Key:    aws.String(key),
+		})
+		if err == nil {
+			return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrAlreadyExist, nil)
+		}
+		if !isNotFound(err) {
+			return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
+		}
+	}
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, err)
+	}
+	if opts.SizeKnown && int64(len(data)) != opts.Size {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", len(data), opts.Size))
+	}
+	contentType := opts.ContentType
+	if contentType == "" {
+		contentType = ContentType(logicalPath)
+	}
+	_, err = b.client.PutObject(ctx, &awss3.PutObjectInput{
+		Bucket:        aws.String(b.bucket),
+		Key:           aws.String(key),
+		Body:          bytes.NewReader(data),
+		ContentLength: aws.Int64(int64(len(data))),
+		ContentType:   aws.String(contentType),
+	})
+	if err != nil {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
+	}
+	return storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: int64(len(data))}, nil
+}
+
+func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	key, err := b.objectKey(logicalPath, false)
+	if err != nil {
+		return storage.Entry{}, err
+	}
+	output, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
+		Bucket: aws.String(b.bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		return storage.Entry{}, b.translateError(storage.OpStat, logicalPath, err)
+	}
+	size := int64(0)
+	if output.ContentLength != nil {
+		size = *output.ContentLength
+	}
+	return storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: size}, nil
+}
+
+func (b *Backend) Walk(ctx context.Context, logicalPrefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if err := storage.ValidatePrefix(logicalPrefix); err != nil {
+		return err
+	}
+	visited := 0
+	emit := func(entry storage.Entry) error {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if opts.Limit > 0 && visited >= opts.Limit {
+			return storage.ErrStopWalk
+		}
+		visited++
+		if err := fn(entry); err != nil {
+			if errors.Is(err, storage.ErrStopWalk) {
+				return storage.ErrStopWalk
+			}
+			return storage.NewError(storage.OpWalk, BackendName, entry.Path, storage.ErrUnknown, err)
+		}
+		return nil
+	}
+	if logicalPrefix != "" {
+		entry, err := b.Stat(ctx, logicalPrefix)
+		if err == nil {
+			if err := emit(entry); errors.Is(err, storage.ErrStopWalk) {
+				return nil
+			} else if err != nil {
+				return err
+			}
+			return nil
+		}
+		if !storage.IsNotFound(err) {
+			return err
+		}
+	}
+	err := b.walkObjects(ctx, logicalPrefix, opts, emit)
+	if errors.Is(err, storage.ErrStopWalk) {
+		return nil
+	}
+	return err
+}
+
+func (b *Backend) HasAny(ctx context.Context, logicalPrefix string) (bool, error) {
+	found := false
+	err := b.Walk(ctx, logicalPrefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
+		found = true
+		return storage.ErrStopWalk
+	})
+	if err != nil {
+		return false, err
+	}
+	return found, nil
+}
+
+func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
+	if err != nil {
+		return err
+	}
+	for _, target := range targets {
+		key, err := b.objectKey(target, false)
+		if err != nil {
+			return err
+		}
+		if key == b.prefix {
+			return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrInvalidPath, nil)
+		}
+		if !opts.IgnoreMissing {
+			_, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
+				Bucket: aws.String(b.bucket),
+				Key:    aws.String(key),
+			})
+			if err != nil {
+				return b.translateError(storage.OpDeleteManagedBundle, target, err)
+			}
+		}
+		_, err = b.client.DeleteObject(ctx, &awss3.DeleteObjectInput{
+			Bucket: aws.String(b.bucket),
+			Key:    aws.String(key),
+		})
+		if err != nil {
+			if opts.IgnoreMissing && isNotFound(err) {
+				continue
+			}
+			return b.translateError(storage.OpDeleteManagedBundle, target, err)
+		}
+	}
+	return nil
+}
+
+func (b *Backend) walkObjects(ctx context.Context, logicalPrefix string, opts storage.WalkOptions, emit func(storage.Entry) error) error {
+	listPrefix, err := b.listPrefix(logicalPrefix)
+	if err != nil {
+		return err
+	}
+	delimiter := ""
+	if !opts.Recursive {
+		delimiter = "/"
+	}
+	var token *string
+	for {
+		output, err := b.client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
+			Bucket:            aws.String(b.bucket),
+			Prefix:            aws.String(listPrefix),
+			Delimiter:         aws.String(delimiter),
+			ContinuationToken: token,
+		})
+		if err != nil {
+			return b.translateError(storage.OpWalk, logicalPrefix, err)
+		}
+		entries := entriesFromList(logicalPrefix, b.prefix, output)
+		sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path })
+		for _, entry := range entries {
+			if entry.Path == "" {
+				continue
+			}
+			if err := emit(entry); err != nil {
+				return err
+			}
+		}
+		if output.IsTruncated == nil || !*output.IsTruncated {
+			return nil
+		}
+		token = output.NextContinuationToken
+	}
+}
+
+func entriesFromList(logicalPrefix, rootPrefix string, output *awss3.ListObjectsV2Output) []storage.Entry {
+	seen := make(map[string]storage.Entry)
+	for _, object := range output.Contents {
+		if object.Key == nil {
+			continue
+		}
+		logicalPath := logicalPathFromKey(rootPrefix, *object.Key)
+		if logicalPath == "" || logicalPath == logicalPrefix {
+			continue
+		}
+		size := int64(0)
+		if object.Size != nil {
+			size = *object.Size
+		}
+		seen[logicalPath] = storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: size}
+	}
+	for _, commonPrefix := range output.CommonPrefixes {
+		if commonPrefix.Prefix == nil {
+			continue
+		}
+		logicalPath := strings.TrimSuffix(logicalPathFromKey(rootPrefix, *commonPrefix.Prefix), "/")
+		if logicalPath == "" || logicalPath == logicalPrefix {
+			continue
+		}
+		seen[logicalPath] = storage.Entry{Path: logicalPath, Type: storage.EntryTypeDirectory}
+	}
+	entries := make([]storage.Entry, 0, len(seen))
+	for _, entry := range seen {
+		entries = append(entries, entry)
+	}
+	return entries
+}
+
+func (b *Backend) objectKey(logicalPath string, allowEmpty bool) (string, error) {
+	if logicalPath == "" {
+		if !allowEmpty {
+			return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
+		}
+		return b.prefix, nil
+	}
+	if err := storage.ValidatePath(logicalPath); err != nil {
+		return "", err
+	}
+	if b.prefix == "" {
+		return logicalPath, nil
+	}
+	return b.prefix + "/" + logicalPath, nil
+}
+
+func (b *Backend) listPrefix(logicalPrefix string) (string, error) {
+	key, err := b.objectKey(logicalPrefix, true)
+	if err != nil {
+		return "", err
+	}
+	if key != "" {
+		key = strings.TrimSuffix(key, "/") + "/"
+	}
+	return key, nil
+}
+
+func logicalPathFromKey(rootPrefix, key string) string {
+	if rootPrefix == "" {
+		return key
+	}
+	if key == rootPrefix {
+		return ""
+	}
+	return strings.TrimPrefix(key, rootPrefix+"/")
+}
+
+func ContentType(logicalPath string) string {
+	switch strings.ToLower(path.Ext(logicalPath)) {
+	case ".md":
+		return "text/markdown; charset=utf-8"
+	case ".html":
+		return "text/html; charset=utf-8"
+	case ".json":
+		return "application/json"
+	case ".txt":
+		return "text/plain; charset=utf-8"
+	default:
+		return "application/octet-stream"
+	}
+}
+
+func isNotFound(err error) bool {
+	var notFound *types.NotFound
+	if errors.As(err, &notFound) {
+		return true
+	}
+	var apiErr smithy.APIError
+	if errors.As(err, &apiErr) {
+		switch apiErr.ErrorCode() {
+		case "NotFound", "NoSuchKey", "404":
+			return true
+		}
+	}
+	return false
+}
+
+func (b *Backend) translateError(op, logicalPath string, err error) error {
+	kind := storage.ErrUnknown
+	if isNotFound(err) {
+		kind = storage.ErrNotFound
+	} else {
+		var apiErr smithy.APIError
+		if errors.As(err, &apiErr) {
+			switch apiErr.ErrorCode() {
+			case "AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch":
+				kind = storage.ErrPermission
+			case "SlowDown", "RequestTimeout", "ServiceUnavailable", "InternalError":
+				kind = storage.ErrTemporary
+			case "InvalidBucketName", "NoSuchBucket":
+				kind = storage.ErrInvalidPath
+			}
+		}
+	}
+	return storage.NewError(op, BackendName, logicalPath, kind, err)
+}