Add remote storage backend
This commit is contained in:
241
internal/adapters/storage/s3_backend.go
Normal file
241
internal/adapters/storage/s3_backend.go
Normal file
@@ -0,0 +1,241 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type s3API interface {
|
||||
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
||||
}
|
||||
|
||||
// S3Backend is an ObjectStore implementation backed by S3-compatible APIs.
|
||||
type S3Backend struct {
|
||||
bucket string
|
||||
client s3API
|
||||
}
|
||||
|
||||
type s3ClientOptions struct {
|
||||
Region string
|
||||
Endpoint string
|
||||
ForcePathStyle bool
|
||||
}
|
||||
|
||||
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
||||
loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1)
|
||||
if strings.TrimSpace(opts.Region) != "" {
|
||||
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
||||
}
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load aws config: %w", err)
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
if strings.TrimSpace(opts.Endpoint) != "" {
|
||||
endpoint := strings.TrimSpace(opts.Endpoint)
|
||||
o.BaseEndpoint = &endpoint
|
||||
}
|
||||
o.UsePathStyle = opts.ForcePathStyle
|
||||
}), nil
|
||||
}
|
||||
|
||||
// NewS3BackendFromConfig builds an S3 backend from resolved config.
|
||||
func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) {
|
||||
bucket := strings.TrimSpace(cfg.Bucket)
|
||||
if bucket == "" {
|
||||
return nil, fmt.Errorf("storage.s3.bucket is required")
|
||||
}
|
||||
|
||||
client, err := newS3Client(ctx, s3ClientOptions{
|
||||
Region: cfg.Region,
|
||||
Endpoint: cfg.Endpoint,
|
||||
ForcePathStyle: cfg.ForcePathStyle,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build s3 client: %w", err)
|
||||
}
|
||||
|
||||
return &S3Backend{
|
||||
bucket: bucket,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// List returns objects under prefix.
|
||||
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||
normalizedPrefix := normalizeObjectKey(prefix)
|
||||
out := make([]ObjectInfo, 0)
|
||||
var token *string
|
||||
|
||||
for {
|
||||
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &b.bucket,
|
||||
Prefix: &normalizedPrefix,
|
||||
ContinuationToken: token,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err)
|
||||
}
|
||||
|
||||
for _, item := range resp.Contents {
|
||||
var lastModified *time.Time
|
||||
if item.LastModified != nil {
|
||||
t := *item.LastModified
|
||||
lastModified = &t
|
||||
}
|
||||
out = append(out, ObjectInfo{
|
||||
Key: normalizeObjectKey(valueOrEmpty(item.Key)),
|
||||
Size: valueOrZeroInt64(item.Size),
|
||||
ETag: strings.Trim(valueOrEmpty(item.ETag), "\""),
|
||||
LastModified: lastModified,
|
||||
})
|
||||
}
|
||||
|
||||
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
|
||||
break
|
||||
}
|
||||
token = resp.NextContinuationToken
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Download retrieves one object to localPath, creating parent directories as needed.
|
||||
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if strings.TrimSpace(localPath) == "" {
|
||||
return fmt.Errorf("download object: local path is required")
|
||||
}
|
||||
|
||||
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: %w", normalizedKey, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
|
||||
}
|
||||
dst, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
if _, err := io.Copy(dst, resp.Body); err != nil {
|
||||
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
|
||||
}
|
||||
if err := dst.Sync(); err != nil {
|
||||
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Upload sends a local file to key.
|
||||
func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if strings.TrimSpace(localPath) == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
|
||||
}
|
||||
if normalizedKey == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||
}
|
||||
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
|
||||
}
|
||||
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
Body: file,
|
||||
Metadata: copyMetadata(opts.Metadata),
|
||||
}
|
||||
if strings.TrimSpace(opts.ContentType) != "" {
|
||||
ct := strings.TrimSpace(opts.ContentType)
|
||||
input.ContentType = &ct
|
||||
}
|
||||
|
||||
resp, err := b.client.PutObject(ctx, input)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||
}
|
||||
|
||||
return ObjectInfo{
|
||||
Key: normalizedKey,
|
||||
Size: stat.Size(),
|
||||
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Exists checks whether one object key exists.
|
||||
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
_, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var notFound *types.NotFound
|
||||
if errors.As(err, ¬Found) {
|
||||
return false, nil
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
switch apiErr.ErrorCode() {
|
||||
case "NotFound", "NoSuchKey", "404":
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
||||
}
|
||||
|
||||
func valueOrEmpty(v *string) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
func valueOrZeroInt64(v *int64) int64 {
|
||||
if v == nil {
|
||||
return 0
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
func valueOrFalseBool(v *bool) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
return *v
|
||||
}
|
||||
Reference in New Issue
Block a user