373 lines
11 KiB
Go
373 lines
11 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/aws/smithy-go"
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
|
)
|
|
|
|
type s3API interface {
|
|
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
|
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
|
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
|
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
|
}
|
|
|
|
// S3Backend is an ObjectStore implementation backed by S3-compatible APIs.
|
|
type S3Backend struct {
|
|
bucket string
|
|
client s3API
|
|
}
|
|
|
|
type s3ClientOptions struct {
|
|
Region string
|
|
Endpoint string
|
|
ForcePathStyle bool
|
|
AccessKeyID string
|
|
SecretKey string
|
|
}
|
|
|
|
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
|
loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1)
|
|
if strings.TrimSpace(opts.Region) != "" {
|
|
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
|
}
|
|
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
|
|
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
|
credentials.NewStaticCredentialsProvider(
|
|
strings.TrimSpace(opts.AccessKeyID),
|
|
strings.TrimSpace(opts.SecretKey),
|
|
"",
|
|
),
|
|
))
|
|
}
|
|
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load aws config: %w", err)
|
|
}
|
|
|
|
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
|
if strings.TrimSpace(opts.Endpoint) != "" {
|
|
endpoint := strings.TrimSpace(opts.Endpoint)
|
|
o.BaseEndpoint = &endpoint
|
|
}
|
|
o.UsePathStyle = opts.ForcePathStyle
|
|
}), nil
|
|
}
|
|
|
|
// NewS3BackendFromConfig builds an S3 backend from resolved config.
|
|
func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) {
|
|
bucket := strings.TrimSpace(cfg.Bucket)
|
|
if bucket == "" {
|
|
return nil, fmt.Errorf("storage.s3.bucket is required")
|
|
}
|
|
|
|
client, err := newS3Client(ctx, s3ClientOptions{
|
|
Region: cfg.Region,
|
|
Endpoint: cfg.Endpoint,
|
|
ForcePathStyle: cfg.ForcePathStyle,
|
|
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
|
|
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("build s3 client: %w", err)
|
|
}
|
|
|
|
return &S3Backend{
|
|
bucket: bucket,
|
|
client: client,
|
|
}, nil
|
|
}
|
|
|
|
func s3CredentialFromEnv(envVarName string) string {
|
|
name := strings.TrimSpace(envVarName)
|
|
if name == "" {
|
|
return ""
|
|
}
|
|
value, ok := os.LookupEnv(name)
|
|
if !ok {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(value)
|
|
}
|
|
|
|
func orDefaultEnvName(name, fallback string) string {
|
|
trimmed := strings.TrimSpace(name)
|
|
if trimmed == "" {
|
|
return fallback
|
|
}
|
|
return trimmed
|
|
}
|
|
|
|
// List returns objects under prefix.
|
|
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
|
normalizedPrefix := normalizeObjectKey(prefix)
|
|
out := make([]ObjectInfo, 0)
|
|
var token *string
|
|
seenTokens := map[string]struct{}{}
|
|
|
|
for {
|
|
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &b.bucket,
|
|
Prefix: &normalizedPrefix,
|
|
ContinuationToken: token,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err)
|
|
}
|
|
|
|
for _, item := range resp.Contents {
|
|
var lastModified *time.Time
|
|
if item.LastModified != nil {
|
|
t := *item.LastModified
|
|
lastModified = &t
|
|
}
|
|
out = append(out, ObjectInfo{
|
|
Key: normalizeObjectKey(valueOrEmpty(item.Key)),
|
|
Size: valueOrZeroInt64(item.Size),
|
|
ETag: strings.Trim(valueOrEmpty(item.ETag), "\""),
|
|
LastModified: lastModified,
|
|
})
|
|
}
|
|
|
|
if !valueOrFalseBool(resp.IsTruncated) {
|
|
break
|
|
}
|
|
next := strings.TrimSpace(valueOrEmpty(resp.NextContinuationToken))
|
|
if next == "" {
|
|
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response has an empty continuation token", b.bucket, normalizedPrefix)
|
|
}
|
|
if _, repeated := seenTokens[next]; repeated {
|
|
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response repeated continuation token", b.bucket, normalizedPrefix)
|
|
}
|
|
seenTokens[next] = struct{}{}
|
|
token = &next
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Read retrieves an object together with the generation observed for its body.
|
|
func (b *S3Backend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
|
|
normalizedKey := normalizeObjectKey(key)
|
|
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &b.bucket, Key: &normalizedKey})
|
|
if err != nil {
|
|
if isS3NotFound(err) {
|
|
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
|
|
}
|
|
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, err)
|
|
}
|
|
var lastModified *time.Time
|
|
if resp.LastModified != nil {
|
|
t := *resp.LastModified
|
|
lastModified = &t
|
|
}
|
|
return ObjectInfo{
|
|
Key: normalizedKey, Size: valueOrZeroInt64(resp.ContentLength),
|
|
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), LastModified: lastModified,
|
|
}, resp.Body, nil
|
|
}
|
|
|
|
// DownloadTo retrieves one object into the caller-owned destination writer.
|
|
func (b *S3Backend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
|
|
if destination == nil {
|
|
return fmt.Errorf("download object: destination writer is required")
|
|
}
|
|
_, body, err := b.Read(ctx, key)
|
|
if err != nil {
|
|
return fmt.Errorf("download object %q: %w", normalizeObjectKey(key), err)
|
|
}
|
|
defer body.Close()
|
|
|
|
if _, err := io.Copy(destination, body); err != nil {
|
|
return fmt.Errorf("download object %q: copy body: %w", normalizeObjectKey(key), err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Download retrieves one object to localPath, creating parent directories as needed.
|
|
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
|
|
if strings.TrimSpace(localPath) == "" {
|
|
return fmt.Errorf("download object: local path is required")
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
|
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
|
}
|
|
dst, err := os.Create(localPath)
|
|
if err != nil {
|
|
return fmt.Errorf("download object %q: create local file: %w", key, err)
|
|
}
|
|
defer dst.Close()
|
|
if err := b.DownloadTo(ctx, key, dst); err != nil {
|
|
return err
|
|
}
|
|
if err := dst.Sync(); err != nil {
|
|
return fmt.Errorf("download object %q: sync local file: %w", key, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Upload sends a local file to key.
|
|
func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
|
|
normalizedKey := normalizeObjectKey(key)
|
|
if strings.TrimSpace(localPath) == "" {
|
|
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
|
|
}
|
|
if normalizedKey == "" {
|
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
|
}
|
|
|
|
file, err := os.Open(localPath)
|
|
if err != nil {
|
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
|
}
|
|
defer file.Close()
|
|
|
|
stat, err := file.Stat()
|
|
if err != nil {
|
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
|
|
}
|
|
return b.uploadReader(ctx, file, key, opts, stat.Size(), WriteCondition{})
|
|
}
|
|
|
|
// UploadReader sends caller-owned content to key.
|
|
func (b *S3Backend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
|
|
return b.uploadReader(ctx, source, key, opts, 0, WriteCondition{})
|
|
}
|
|
|
|
// UploadConditional uploads a mutable object only when its observed generation
|
|
// still matches, or when no object exists yet.
|
|
func (b *S3Backend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
|
|
if err := validateWriteCondition(condition); err != nil {
|
|
return ObjectInfo{}, err
|
|
}
|
|
return b.uploadReader(ctx, source, key, opts, 0, condition)
|
|
}
|
|
|
|
func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64, condition WriteCondition) (ObjectInfo, error) {
|
|
normalizedKey := normalizeObjectKey(key)
|
|
if source == nil {
|
|
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
|
|
}
|
|
if normalizedKey == "" {
|
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
|
}
|
|
|
|
input := &s3.PutObjectInput{
|
|
Bucket: &b.bucket,
|
|
Key: &normalizedKey,
|
|
Body: source,
|
|
Metadata: copyMetadata(opts.Metadata),
|
|
}
|
|
if strings.TrimSpace(opts.ContentType) != "" {
|
|
ct := strings.TrimSpace(opts.ContentType)
|
|
input.ContentType = &ct
|
|
}
|
|
if condition.RequireAbsent {
|
|
wildcard := "*"
|
|
input.IfNoneMatch = &wildcard
|
|
} else if expected := strings.TrimSpace(condition.MatchETag); expected != "" {
|
|
input.IfMatch = &expected
|
|
}
|
|
|
|
resp, err := b.client.PutObject(ctx, input)
|
|
if err != nil {
|
|
if isS3ConditionalConflict(err) {
|
|
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, ErrConditionNotMet)
|
|
}
|
|
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
|
|
}
|
|
|
|
info := ObjectInfo{
|
|
Key: normalizedKey,
|
|
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
|
|
}
|
|
if size > 0 {
|
|
info.Size = size
|
|
}
|
|
return info, nil
|
|
}
|
|
|
|
// Exists checks whether one object key exists.
|
|
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
|
normalizedKey := normalizeObjectKey(key)
|
|
_, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: &b.bucket,
|
|
Key: &normalizedKey,
|
|
})
|
|
if err == nil {
|
|
return true, nil
|
|
}
|
|
|
|
if isS3NotFound(err) {
|
|
return false, nil
|
|
}
|
|
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
|
}
|
|
|
|
func validateWriteCondition(condition WriteCondition) error {
|
|
if condition.RequireAbsent == (strings.TrimSpace(condition.MatchETag) != "") {
|
|
return fmt.Errorf("conditional upload requires exactly one of MatchETag or RequireAbsent")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func isS3NotFound(err error) bool {
|
|
var notFound *types.NotFound
|
|
if errors.As(err, ¬Found) {
|
|
return true
|
|
}
|
|
var apiErr smithy.APIError
|
|
if errors.As(err, &apiErr) {
|
|
switch apiErr.ErrorCode() {
|
|
case "NotFound", "NoSuchKey", "404":
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isS3ConditionalConflict(err error) bool {
|
|
var apiErr smithy.APIError
|
|
if errors.As(err, &apiErr) {
|
|
switch apiErr.ErrorCode() {
|
|
case "PreconditionFailed", "ConditionalRequestConflict", "412", "409":
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func valueOrEmpty(v *string) string {
|
|
if v == nil {
|
|
return ""
|
|
}
|
|
return *v
|
|
}
|
|
|
|
func valueOrZeroInt64(v *int64) int64 {
|
|
if v == nil {
|
|
return 0
|
|
}
|
|
return *v
|
|
}
|
|
|
|
func valueOrFalseBool(v *bool) bool {
|
|
if v == nil {
|
|
return false
|
|
}
|
|
return *v
|
|
}
|