eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add S3-compatible storage backend

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-31 17:11:53 +00:00
parent 052aa8a64a
commit 14fa9c8000
27 changed files with 1334 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
internal/app/backends.go
View File

@@ -6,6 +6,7 @@ import (
"strconv"
"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
s3adapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/s3"
sshadapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/ssh"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
@@ -20,6 +21,13 @@ const (
sshKeyFileKey = "ssh_key_file"
sshKnownHostsKey = "known_hosts"
sshHostKeyPolicyKey = "host_key_policy"
s3EndpointKey = "endpoint"
s3BucketKey = "bucket"
s3PrefixKey = "prefix"
s3RegionKey = "region"
s3ForcePathStyleKey = "force_path_style"
s3AccessKeyIDKey = "access_key_id"
s3SecretAccessKey = "secret_access_key"
)
type backendFactory struct {
@@ -54,21 +62,44 @@ func newBackendFactoryWithEnvironment(environment config.Environment) *backendFa
HostKeyPolicy: sshadapter.HostKeyPolicy(cfg[sshHostKeyPolicyKey]),
})
})
_ = registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
forcePathStyle, err := strconv.ParseBool(cfg[s3ForcePathStyleKey])
if err != nil {
return nil, fmt.Errorf("s3 force_path_style: %w", err)
}
return s3adapter.New(ctx, s3adapter.Options{
Endpoint: cfg[s3EndpointKey],
Bucket: cfg[s3BucketKey],
Prefix: cfg[s3PrefixKey],
Region: cfg[s3RegionKey],
ForcePathStyle: forcePathStyle,
AccessKeyID: cfg[s3AccessKeyIDKey],
SecretAccessKey: cfg[s3SecretAccessKey],
})
})
return &backendFactory{registry: registry, environment: environment}
}
func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) {
if source.Backend != config.BackendLocal && source.Backend != config.BackendSSH {
if source.Backend != config.BackendLocal && source.Backend != config.BackendSSH && source.Backend != config.BackendS3 {
return nil, fmt.Errorf("source backend %s is not implemented for execution", source.Backend)
}
return f.registry.Open(ctx, source.Backend, sourceOpenConfig(source))
openConfig, err := f.sourceOpenConfig(source)
if err != nil {
return nil, err
}
return f.registry.Open(ctx, source.Backend, openConfig)
}
func (f *backendFactory) openDestination(ctx context.Context, destination config.Destination) (storage.Backend, error) {
if destination.Backend != config.BackendLocal && destination.Backend != config.BackendSSH {
if destination.Backend != config.BackendLocal && destination.Backend != config.BackendSSH && destination.Backend != config.BackendS3 {
return nil, fmt.Errorf("backend %s is not implemented for execution", destination.Backend)
}
return f.registry.Open(ctx, destination.Backend, destinationOpenConfig(destination))
openConfig, err := f.destinationOpenConfig(destination)
if err != nil {
return nil, err
}
return f.registry.Open(ctx, destination.Backend, openConfig)
}
func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storage.Backend, error) {
@@ -79,6 +110,43 @@ func (f *backendFactory) resolveCredentials(creds config.Credentials) (config.Re
return f.environment.ResolveCredentials(creds)
}
func (f *backendFactory) sourceOpenConfig(source config.Backend) (storage.OpenConfig, error) {
cfg := sourceOpenConfig(source)
if source.Backend == config.BackendS3 {
if err := f.addS3Config(cfg, source.Endpoint, source.Bucket, source.Prefix, source.Region, source.ForcePath, source.Creds); err != nil {
return nil, err
}
}
return cfg, nil
}
func (f *backendFactory) destinationOpenConfig(destination config.Destination) (storage.OpenConfig, error) {
cfg := destinationOpenConfig(destination)
if destination.Backend == config.BackendS3 {
if err := f.addS3Config(cfg, destination.Endpoint, destination.Bucket, destination.Prefix, destination.Region, destination.ForcePath, destination.Creds); err != nil {
return nil, err
}
}
return cfg, nil
}
func (f *backendFactory) addS3Config(cfg storage.OpenConfig, endpoint, bucket, prefix, region string, forcePath *bool, creds config.Credentials) error {
cfg[s3EndpointKey] = endpoint
cfg[s3BucketKey] = bucket
cfg[s3PrefixKey] = prefix
cfg[s3RegionKey] = region
cfg[s3ForcePathStyleKey] = strconv.FormatBool(config.ForcePathStyle(forcePath))
if creds.AccessKeyIDEnv != "" || creds.SecretAccessKeyEnv != "" {
resolved, err := f.resolveCredentials(creds)
if err != nil {
return err
}
cfg[s3AccessKeyIDKey] = resolved.AccessKeyID
cfg[s3SecretAccessKey] = resolved.SecretAccessKey
}
return nil
}
func sourceOpenConfig(source config.Backend) storage.OpenConfig {
cfg := storage.OpenConfig{storagePathKey: source.Path}
if source.Backend == config.BackendSSH {

84
internal/app/backends_test.go
View File

@@ -110,11 +110,9 @@ func TestBackendFactoryOpensSSHDestinationWithRegisteredOpener(t *testing.T) {
func TestBackendFactoryRejectsUnsupportedSource(t *testing.T) {
factory := newBackendFactory()
_, err := factory.openSource(context.Background(), config.Backend{
Backend: config.BackendS3,
Endpoint: "https://s3.example.com",
Bucket: "reports",
Backend: "ftp",
})
if err == nil || !strings.Contains(err.Error(), "source backend s3 is not implemented for execution") {
if err == nil || !strings.Contains(err.Error(), "source backend ftp is not implemented for execution") {
t.Fatalf("openSource() error = %v, want not implemented", err)
}
}
@@ -122,15 +120,85 @@ func TestBackendFactoryRejectsUnsupportedSource(t *testing.T) {
func TestBackendFactoryRejectsUnsupportedDestination(t *testing.T) {
factory := newBackendFactory()
_, err := factory.openDestination(context.Background(), config.Destination{
Backend: config.BackendS3,
Endpoint: "https://s3.example.com",
Bucket: "reports",
Backend: "ftp",
})
if err == nil || !strings.Contains(err.Error(), "backend s3 is not implemented for execution") {
if err == nil || !strings.Contains(err.Error(), "backend ftp is not implemented for execution") {
t.Fatalf("openDestination() error = %v, want not implemented", err)
}
}
func TestBackendFactoryOpensS3DestinationWithRegisteredOpener(t *testing.T) {
factory := &backendFactory{
registry: storage.NewRegistry(),
environment: config.NewEnvironment(nil, func(string) (string, bool) { return "", false }),
}
var got storage.OpenConfig
if err := factory.registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
got = cfg
return fake.New(), nil
}); err != nil {
t.Fatalf("Register() error = %v", err)
}
forcePathStyle := false
backend, err := factory.openDestination(context.Background(), config.Destination{
Backend: config.BackendS3,
Endpoint: "https://s3.example.com",
Bucket: "reports",
Prefix: "archive",
Region: config.DefaultS3Region,
ForcePath: &forcePathStyle,
})
if err != nil {
t.Fatalf("openDestination() error = %v", err)
}
if backend == nil {
t.Fatal("openDestination() backend = nil")
}
assertOpenConfig(t, got, map[string]string{
s3EndpointKey: "https://s3.example.com",
s3BucketKey: "reports",
s3PrefixKey: "archive",
s3RegionKey: config.DefaultS3Region,
s3ForcePathStyleKey: "false",
})
}
func TestBackendFactoryResolvesS3CredentialsThroughSecretsAwareEnvironment(t *testing.T) {
factory := &backendFactory{
registry: storage.NewRegistry(),
environment: config.NewEnvironment(map[string]string{
"ACCESS_KEY_ID": "secret-access",
"SECRET_ACCESS_KEY": "secret-secret",
}, func(string) (string, bool) { return "", false }),
}
var got storage.OpenConfig
if err := factory.registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
got = cfg
return fake.New(), nil
}); err != nil {
t.Fatalf("Register() error = %v", err)
}
forcePathStyle := true
_, err := factory.openSource(context.Background(), config.Backend{
Backend: config.BackendS3,
Endpoint: "https://s3.example.com",
Bucket: "reports",
Region: config.DefaultS3Region,
ForcePath: &forcePathStyle,
Creds: config.Credentials{
AccessKeyIDEnv: "ACCESS_KEY_ID",
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
},
})
if err != nil {
t.Fatalf("openSource() error = %v", err)
}
assertOpenConfig(t, got, map[string]string{
s3AccessKeyIDKey: "secret-access",
s3SecretAccessKey: "secret-secret",
})
}
func TestBackendFactoryResolvesCredentialsThroughEnvironment(t *testing.T) {
factory := newBackendFactoryWithEnvironment(config.NewEnvironment(map[string]string{
"ACCESS_KEY_ID": "secret-access",