Add S3-compatible storage backend

internal/app/backends.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 
 	"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
+	s3adapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/s3"
 	sshadapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/ssh"
 	"gitea.maximumdirect.net/eric/distributor/internal/config"
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
@@ -20,6 +21,13 @@ const (
 	sshKeyFileKey       = "ssh_key_file"
 	sshKnownHostsKey    = "known_hosts"
 	sshHostKeyPolicyKey = "host_key_policy"
+	s3EndpointKey       = "endpoint"
+	s3BucketKey         = "bucket"
+	s3PrefixKey         = "prefix"
+	s3RegionKey         = "region"
+	s3ForcePathStyleKey = "force_path_style"
+	s3AccessKeyIDKey    = "access_key_id"
+	s3SecretAccessKey   = "secret_access_key"
 )
 
 type backendFactory struct {
@@ -54,21 +62,44 @@ func newBackendFactoryWithEnvironment(environment config.Environment) *backendFa
 			HostKeyPolicy: sshadapter.HostKeyPolicy(cfg[sshHostKeyPolicyKey]),
 		})
 	})
+	_ = registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		forcePathStyle, err := strconv.ParseBool(cfg[s3ForcePathStyleKey])
+		if err != nil {
+			return nil, fmt.Errorf("s3 force_path_style: %w", err)
+		}
+		return s3adapter.New(ctx, s3adapter.Options{
+			Endpoint:        cfg[s3EndpointKey],
+			Bucket:          cfg[s3BucketKey],
+			Prefix:          cfg[s3PrefixKey],
+			Region:          cfg[s3RegionKey],
+			ForcePathStyle:  forcePathStyle,
+			AccessKeyID:     cfg[s3AccessKeyIDKey],
+			SecretAccessKey: cfg[s3SecretAccessKey],
+		})
+	})
 	return &backendFactory{registry: registry, environment: environment}
 }
 
 func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) {
-	if source.Backend != config.BackendLocal && source.Backend != config.BackendSSH {
+	if source.Backend != config.BackendLocal && source.Backend != config.BackendSSH && source.Backend != config.BackendS3 {
 		return nil, fmt.Errorf("source backend %s is not implemented for execution", source.Backend)
 	}
-	return f.registry.Open(ctx, source.Backend, sourceOpenConfig(source))
+	openConfig, err := f.sourceOpenConfig(source)
+	if err != nil {
+		return nil, err
+	}
+	return f.registry.Open(ctx, source.Backend, openConfig)
 }
 
 func (f *backendFactory) openDestination(ctx context.Context, destination config.Destination) (storage.Backend, error) {
-	if destination.Backend != config.BackendLocal && destination.Backend != config.BackendSSH {
+	if destination.Backend != config.BackendLocal && destination.Backend != config.BackendSSH && destination.Backend != config.BackendS3 {
 		return nil, fmt.Errorf("backend %s is not implemented for execution", destination.Backend)
 	}
-	return f.registry.Open(ctx, destination.Backend, destinationOpenConfig(destination))
+	openConfig, err := f.destinationOpenConfig(destination)
+	if err != nil {
+		return nil, err
+	}
+	return f.registry.Open(ctx, destination.Backend, openConfig)
 }
 
 func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storage.Backend, error) {
@@ -79,6 +110,43 @@ func (f *backendFactory) resolveCredentials(creds config.Credentials) (config.Re
 	return f.environment.ResolveCredentials(creds)
 }
 
+func (f *backendFactory) sourceOpenConfig(source config.Backend) (storage.OpenConfig, error) {
+	cfg := sourceOpenConfig(source)
+	if source.Backend == config.BackendS3 {
+		if err := f.addS3Config(cfg, source.Endpoint, source.Bucket, source.Prefix, source.Region, source.ForcePath, source.Creds); err != nil {
+			return nil, err
+		}
+	}
+	return cfg, nil
+}
+
+func (f *backendFactory) destinationOpenConfig(destination config.Destination) (storage.OpenConfig, error) {
+	cfg := destinationOpenConfig(destination)
+	if destination.Backend == config.BackendS3 {
+		if err := f.addS3Config(cfg, destination.Endpoint, destination.Bucket, destination.Prefix, destination.Region, destination.ForcePath, destination.Creds); err != nil {
+			return nil, err
+		}
+	}
+	return cfg, nil
+}
+
+func (f *backendFactory) addS3Config(cfg storage.OpenConfig, endpoint, bucket, prefix, region string, forcePath *bool, creds config.Credentials) error {
+	cfg[s3EndpointKey] = endpoint
+	cfg[s3BucketKey] = bucket
+	cfg[s3PrefixKey] = prefix
+	cfg[s3RegionKey] = region
+	cfg[s3ForcePathStyleKey] = strconv.FormatBool(config.ForcePathStyle(forcePath))
+	if creds.AccessKeyIDEnv != "" || creds.SecretAccessKeyEnv != "" {
+		resolved, err := f.resolveCredentials(creds)
+		if err != nil {
+			return err
+		}
+		cfg[s3AccessKeyIDKey] = resolved.AccessKeyID
+		cfg[s3SecretAccessKey] = resolved.SecretAccessKey
+	}
+	return nil
+}
+
 func sourceOpenConfig(source config.Backend) storage.OpenConfig {
 	cfg := storage.OpenConfig{storagePathKey: source.Path}
 	if source.Backend == config.BackendSSH {