package storage import ( "context" "errors" "fmt" "io" "os" "path/filepath" "strings" "time" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" "gitea.maximumdirect.net/eric/narratio/internal/config" ) type s3API interface { ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) } // S3Backend is an ObjectStore implementation backed by S3-compatible APIs. type S3Backend struct { bucket string client s3API } type s3ClientOptions struct { Region string Endpoint string ForcePathStyle bool AccessKeyID string SecretKey string } var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) { loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1) if strings.TrimSpace(opts.Region) != "" { loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region))) } if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" { loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider( credentials.NewStaticCredentialsProvider( strings.TrimSpace(opts.AccessKeyID), strings.TrimSpace(opts.SecretKey), "", ), )) } awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...) if err != nil { return nil, fmt.Errorf("load aws config: %w", err) } return s3.NewFromConfig(awsCfg, func(o *s3.Options) { if strings.TrimSpace(opts.Endpoint) != "" { endpoint := strings.TrimSpace(opts.Endpoint) o.BaseEndpoint = &endpoint } o.UsePathStyle = opts.ForcePathStyle }), nil } // NewS3BackendFromConfig builds an S3 backend from resolved config. func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) { bucket := strings.TrimSpace(cfg.Bucket) if bucket == "" { return nil, fmt.Errorf("storage.s3.bucket is required") } client, err := newS3Client(ctx, s3ClientOptions{ Region: cfg.Region, Endpoint: cfg.Endpoint, ForcePathStyle: cfg.ForcePathStyle, AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)), SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)), }) if err != nil { return nil, fmt.Errorf("build s3 client: %w", err) } return &S3Backend{ bucket: bucket, client: client, }, nil } func s3CredentialFromEnv(envVarName string) string { name := strings.TrimSpace(envVarName) if name == "" { return "" } value, ok := os.LookupEnv(name) if !ok { return "" } return strings.TrimSpace(value) } func orDefaultEnvName(name, fallback string) string { trimmed := strings.TrimSpace(name) if trimmed == "" { return fallback } return trimmed } // List returns objects under prefix. func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) { normalizedPrefix := normalizeObjectKey(prefix) out := make([]ObjectInfo, 0) var token *string for { resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: &b.bucket, Prefix: &normalizedPrefix, ContinuationToken: token, }) if err != nil { return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err) } for _, item := range resp.Contents { var lastModified *time.Time if item.LastModified != nil { t := *item.LastModified lastModified = &t } out = append(out, ObjectInfo{ Key: normalizeObjectKey(valueOrEmpty(item.Key)), Size: valueOrZeroInt64(item.Size), ETag: strings.Trim(valueOrEmpty(item.ETag), "\""), LastModified: lastModified, }) } if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil { break } token = resp.NextContinuationToken } return out, nil } // DownloadTo retrieves one object into the caller-owned destination writer. func (b *S3Backend) DownloadTo(ctx context.Context, key string, destination io.Writer) error { normalizedKey := normalizeObjectKey(key) if destination == nil { return fmt.Errorf("download object: destination writer is required") } resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: &b.bucket, Key: &normalizedKey, }) if err != nil { return fmt.Errorf("download object %q: %w", normalizedKey, err) } defer resp.Body.Close() if _, err := io.Copy(destination, resp.Body); err != nil { return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err) } return nil } // Download retrieves one object to localPath, creating parent directories as needed. func (b *S3Backend) Download(ctx context.Context, key, localPath string) error { if strings.TrimSpace(localPath) == "" { return fmt.Errorf("download object: local path is required") } if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { return fmt.Errorf("download object %q: create parent directory: %w", key, err) } dst, err := os.Create(localPath) if err != nil { return fmt.Errorf("download object %q: create local file: %w", key, err) } defer dst.Close() if err := b.DownloadTo(ctx, key, dst); err != nil { return err } if err := dst.Sync(); err != nil { return fmt.Errorf("download object %q: sync local file: %w", key, err) } return nil } // Upload sends a local file to key. func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) { normalizedKey := normalizeObjectKey(key) if strings.TrimSpace(localPath) == "" { return ObjectInfo{}, fmt.Errorf("upload object: local path is required") } if normalizedKey == "" { return ObjectInfo{}, fmt.Errorf("upload object: key is required") } file, err := os.Open(localPath) if err != nil { return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err) } defer file.Close() stat, err := file.Stat() if err != nil { return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err) } input := &s3.PutObjectInput{ Bucket: &b.bucket, Key: &normalizedKey, Body: file, Metadata: copyMetadata(opts.Metadata), } if strings.TrimSpace(opts.ContentType) != "" { ct := strings.TrimSpace(opts.ContentType) input.ContentType = &ct } resp, err := b.client.PutObject(ctx, input) if err != nil { return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err) } return ObjectInfo{ Key: normalizedKey, Size: stat.Size(), ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), }, nil } // Exists checks whether one object key exists. func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) { normalizedKey := normalizeObjectKey(key) _, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: &b.bucket, Key: &normalizedKey, }) if err == nil { return true, nil } var notFound *types.NotFound if errors.As(err, ¬Found) { return false, nil } var apiErr smithy.APIError if errors.As(err, &apiErr) { switch apiErr.ErrorCode() { case "NotFound", "NoSuchKey", "404": return false, nil } } return false, fmt.Errorf("head object %q: %w", normalizedKey, err) } func valueOrEmpty(v *string) string { if v == nil { return "" } return *v } func valueOrZeroInt64(v *int64) int64 { if v == nil { return 0 } return *v } func valueOrFalseBool(v *bool) bool { if v == nil { return false } return *v }