package storage import ( "context" "errors" "fmt" "io" "os" "path/filepath" "strings" "time" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" "gitea.maximumdirect.net/eric/narratio/internal/config" ) type s3API interface { ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) } // S3Backend is an ObjectStore implementation backed by S3-compatible APIs. type S3Backend struct { bucket string client s3API } type s3ClientOptions struct { Region string Endpoint string ForcePathStyle bool AccessKeyID string SecretKey string } var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) { loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1) if strings.TrimSpace(opts.Region) != "" { loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region))) } if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" { loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider( credentials.NewStaticCredentialsProvider( strings.TrimSpace(opts.AccessKeyID), strings.TrimSpace(opts.SecretKey), "", ), )) } awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...) if err != nil { return nil, fmt.Errorf("load aws config: %w", err) } return s3.NewFromConfig(awsCfg, func(o *s3.Options) { if strings.TrimSpace(opts.Endpoint) != "" { endpoint := strings.TrimSpace(opts.Endpoint) o.BaseEndpoint = &endpoint } o.UsePathStyle = opts.ForcePathStyle }), nil } // NewS3BackendFromConfig builds an S3 backend from resolved config. func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) { bucket := strings.TrimSpace(cfg.Bucket) if bucket == "" { return nil, fmt.Errorf("storage.s3.bucket is required") } client, err := newS3Client(ctx, s3ClientOptions{ Region: cfg.Region, Endpoint: cfg.Endpoint, ForcePathStyle: cfg.ForcePathStyle, AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)), SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)), }) if err != nil { return nil, fmt.Errorf("build s3 client: %w", err) } return &S3Backend{ bucket: bucket, client: client, }, nil } func s3CredentialFromEnv(envVarName string) string { name := strings.TrimSpace(envVarName) if name == "" { return "" } value, ok := os.LookupEnv(name) if !ok { return "" } return strings.TrimSpace(value) } func orDefaultEnvName(name, fallback string) string { trimmed := strings.TrimSpace(name) if trimmed == "" { return fallback } return trimmed } // List returns objects under prefix. func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) { normalizedPrefix := normalizeObjectKey(prefix) out := make([]ObjectInfo, 0) var token *string seenTokens := map[string]struct{}{} for { resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: &b.bucket, Prefix: &normalizedPrefix, ContinuationToken: token, }) if err != nil { return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err) } for _, item := range resp.Contents { var lastModified *time.Time if item.LastModified != nil { t := *item.LastModified lastModified = &t } out = append(out, ObjectInfo{ Key: normalizeObjectKey(valueOrEmpty(item.Key)), Size: valueOrZeroInt64(item.Size), ETag: strings.Trim(valueOrEmpty(item.ETag), "\""), LastModified: lastModified, }) } if !valueOrFalseBool(resp.IsTruncated) { break } next := strings.TrimSpace(valueOrEmpty(resp.NextContinuationToken)) if next == "" { return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response has an empty continuation token", b.bucket, normalizedPrefix) } if _, repeated := seenTokens[next]; repeated { return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response repeated continuation token", b.bucket, normalizedPrefix) } seenTokens[next] = struct{}{} token = &next } return out, nil } // Read retrieves an object together with the generation observed for its body. func (b *S3Backend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) { normalizedKey := normalizeObjectKey(key) resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &b.bucket, Key: &normalizedKey}) if err != nil { if isS3NotFound(err) { return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist) } return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, err) } var lastModified *time.Time if resp.LastModified != nil { t := *resp.LastModified lastModified = &t } return ObjectInfo{ Key: normalizedKey, Size: valueOrZeroInt64(resp.ContentLength), ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), LastModified: lastModified, }, resp.Body, nil } // DownloadTo retrieves one object into the caller-owned destination writer. func (b *S3Backend) DownloadTo(ctx context.Context, key string, destination io.Writer) error { if destination == nil { return fmt.Errorf("download object: destination writer is required") } _, body, err := b.Read(ctx, key) if err != nil { return fmt.Errorf("download object %q: %w", normalizeObjectKey(key), err) } defer body.Close() if _, err := io.Copy(destination, body); err != nil { return fmt.Errorf("download object %q: copy body: %w", normalizeObjectKey(key), err) } return nil } // Download retrieves one object to localPath, creating parent directories as needed. func (b *S3Backend) Download(ctx context.Context, key, localPath string) error { if strings.TrimSpace(localPath) == "" { return fmt.Errorf("download object: local path is required") } if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { return fmt.Errorf("download object %q: create parent directory: %w", key, err) } dst, err := os.Create(localPath) if err != nil { return fmt.Errorf("download object %q: create local file: %w", key, err) } defer dst.Close() if err := b.DownloadTo(ctx, key, dst); err != nil { return err } if err := dst.Sync(); err != nil { return fmt.Errorf("download object %q: sync local file: %w", key, err) } return nil } // Upload sends a local file to key. func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) { normalizedKey := normalizeObjectKey(key) if strings.TrimSpace(localPath) == "" { return ObjectInfo{}, fmt.Errorf("upload object: local path is required") } if normalizedKey == "" { return ObjectInfo{}, fmt.Errorf("upload object: key is required") } file, err := os.Open(localPath) if err != nil { return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err) } defer file.Close() stat, err := file.Stat() if err != nil { return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err) } return b.uploadReader(ctx, file, key, opts, stat.Size(), WriteCondition{}) } // UploadReader sends caller-owned content to key. func (b *S3Backend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) { return b.uploadReader(ctx, source, key, opts, 0, WriteCondition{}) } // UploadConditional uploads a mutable object only when its observed generation // still matches, or when no object exists yet. func (b *S3Backend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) { if err := validateWriteCondition(condition); err != nil { return ObjectInfo{}, err } return b.uploadReader(ctx, source, key, opts, 0, condition) } func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64, condition WriteCondition) (ObjectInfo, error) { normalizedKey := normalizeObjectKey(key) if source == nil { return ObjectInfo{}, fmt.Errorf("upload object: source is required") } if normalizedKey == "" { return ObjectInfo{}, fmt.Errorf("upload object: key is required") } input := &s3.PutObjectInput{ Bucket: &b.bucket, Key: &normalizedKey, Body: source, Metadata: copyMetadata(opts.Metadata), } if strings.TrimSpace(opts.ContentType) != "" { ct := strings.TrimSpace(opts.ContentType) input.ContentType = &ct } if condition.RequireAbsent { wildcard := "*" input.IfNoneMatch = &wildcard } else if expected := strings.TrimSpace(condition.MatchETag); expected != "" { input.IfMatch = &expected } resp, err := b.client.PutObject(ctx, input) if err != nil { if isS3ConditionalConflict(err) { return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, ErrConditionNotMet) } return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err) } info := ObjectInfo{ Key: normalizedKey, ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), } if size > 0 { info.Size = size } return info, nil } // Exists checks whether one object key exists. func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) { normalizedKey := normalizeObjectKey(key) _, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: &b.bucket, Key: &normalizedKey, }) if err == nil { return true, nil } if isS3NotFound(err) { return false, nil } return false, fmt.Errorf("head object %q: %w", normalizedKey, err) } func validateWriteCondition(condition WriteCondition) error { if condition.RequireAbsent == (strings.TrimSpace(condition.MatchETag) != "") { return fmt.Errorf("conditional upload requires exactly one of MatchETag or RequireAbsent") } return nil } func isS3NotFound(err error) bool { var notFound *types.NotFound if errors.As(err, ¬Found) { return true } var apiErr smithy.APIError if errors.As(err, &apiErr) { switch apiErr.ErrorCode() { case "NotFound", "NoSuchKey", "404": return true } } return false } func isS3ConditionalConflict(err error) bool { var apiErr smithy.APIError if errors.As(err, &apiErr) { switch apiErr.ErrorCode() { case "PreconditionFailed", "ConditionalRequestConflict", "412", "409": return true } } return false } func valueOrEmpty(v *string) string { if v == nil { return "" } return *v } func valueOrZeroInt64(v *int64) int64 { if v == nil { return 0 } return *v } func valueOrFalseBool(v *bool) bool { if v == nil { return false } return *v }