From 7cf8f74c3e14b005445ad7416644402a51e21158 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 4 Jun 2026 00:31:33 +0000 Subject: [PATCH] Normalize backend config validation --- internal/config/backend_view.go | 50 +++++++ internal/config/backend_view_test.go | 187 +++++++++++++++++++++++++++ internal/config/defaults.go | 30 ++--- internal/config/validate.go | 32 ++--- 4 files changed, 265 insertions(+), 34 deletions(-) create mode 100644 internal/config/backend_view.go create mode 100644 internal/config/backend_view_test.go diff --git a/internal/config/backend_view.go b/internal/config/backend_view.go new file mode 100644 index 0000000..ca3f148 --- /dev/null +++ b/internal/config/backend_view.go @@ -0,0 +1,50 @@ +package config + +type backendView struct { + Backend string + Host string + User string + Port int + Path string + Endpoint string + Bucket string + Prefix string + Region string + ForcePath *bool + Creds Credentials + SSH SSH +} + +func backendViewFromSource(source Backend) backendView { + return backendView{ + Backend: source.Backend, + Host: source.Host, + User: source.User, + Port: source.Port, + Path: source.Path, + Endpoint: source.Endpoint, + Bucket: source.Bucket, + Prefix: source.Prefix, + Region: source.Region, + ForcePath: source.ForcePath, + Creds: source.Creds, + SSH: source.SSH, + } +} + +func backendViewFromDestination(destination Destination) backendView { + return backendView{ + Backend: destination.Backend, + Host: destination.Host, + User: destination.User, + Port: destination.Port, + Path: destination.Path, + Endpoint: destination.Endpoint, + Bucket: destination.Bucket, + Prefix: destination.Prefix, + Region: destination.Region, + ForcePath: destination.ForcePath, + Creds: destination.Creds, + SSH: destination.SSH, + } +} diff --git a/internal/config/backend_view_test.go b/internal/config/backend_view_test.go new file mode 100644 index 0000000..915cf25 --- /dev/null +++ b/internal/config/backend_view_test.go @@ -0,0 +1,187 @@ +package config + +import "testing" + +func TestBackendViewsPreserveEquivalentStorageFields(t *testing.T) { + forcePathStyle := false + source := Backend{ + Backend: BackendS3, + Host: "storage.example.com", + User: "reports", + Port: 2222, + Path: "/reports", + Endpoint: "https://s3.example.com", + Bucket: "source", + Prefix: "incoming", + Region: "us-west-2", + ForcePath: &forcePathStyle, + Creds: Credentials{ + AccessKeyIDEnv: "ACCESS_KEY_ID", + SecretAccessKeyEnv: "SECRET_ACCESS_KEY", + }, + SSH: SSH{ + KeyFile: "/home/reports/.ssh/id_ed25519", + KnownHosts: "/home/reports/.ssh/known_hosts", + HostKeyPolicy: HostKeyPolicyStrict, + }, + } + destination := Destination{ + Backend: source.Backend, + Host: source.Host, + User: source.User, + Port: source.Port, + Path: source.Path, + Endpoint: source.Endpoint, + Bucket: source.Bucket, + Prefix: source.Prefix, + Region: source.Region, + ForcePath: source.ForcePath, + Creds: source.Creds, + SSH: source.SSH, + } + + sourceView := backendViewFromSource(source) + destinationView := backendViewFromDestination(destination) + + if sourceView != destinationView { + t.Fatalf("source view = %#v, destination view = %#v, want equivalent storage fields", sourceView, destinationView) + } +} + +func TestBackendViewValidationKeepsHTTPUploadSourceOnly(t *testing.T) { + cfg := Config{Pipelines: []Pipeline{{ + ID: "reports", + Source: Backend{ + Backend: BackendHTTPUpload, + Upload: HTTPUpload{TokenEnv: "UPLOAD_TOKEN"}, + }, + Destinations: []Destination{{ + ID: "archive", + Backend: BackendHTTPUpload, + }}, + }}} + ApplyDefaults(&cfg) + + err := Validate(cfg) + if err == nil { + t.Fatal("Validate() error = nil, want destination http_upload error") + } + if got, want := err.Error(), "pipelines[0].destinations[0].backend http_upload is only supported for sources"; got != want { + t.Fatalf("Validate() error = %q, want %q", got, want) + } +} + +func TestBackendViewValidationAppliesStorageRulesToSourcesAndDestinations(t *testing.T) { + forcePathStyle := false + tests := []struct { + name string + source Backend + destination Destination + wantErr bool + }{ + { + name: "local valid", + source: Backend{ + Backend: BackendLocal, + Path: "/source", + }, + destination: Destination{ + Backend: BackendLocal, + Path: "/destination", + }, + }, + { + name: "local missing path", + source: Backend{ + Backend: BackendLocal, + }, + destination: Destination{ + Backend: BackendLocal, + }, + wantErr: true, + }, + { + name: "ssh valid", + source: Backend{ + Backend: BackendSSH, + Host: "source.example.com", + Port: 22, + Path: "/source", + SSH: SSH{HostKeyPolicy: HostKeyPolicyAcceptNew}, + }, + destination: Destination{ + Backend: BackendSSH, + Host: "destination.example.com", + Port: 22, + Path: "/destination", + SSH: SSH{HostKeyPolicy: HostKeyPolicyAcceptNew}, + }, + }, + { + name: "ssh missing host", + source: Backend{ + Backend: BackendSSH, + Port: 22, + Path: "/source", + SSH: SSH{HostKeyPolicy: HostKeyPolicyAcceptNew}, + }, + destination: Destination{ + Backend: BackendSSH, + Port: 22, + Path: "/destination", + SSH: SSH{HostKeyPolicy: HostKeyPolicyAcceptNew}, + }, + wantErr: true, + }, + { + name: "s3 valid", + source: Backend{ + Backend: BackendS3, + Endpoint: "https://s3.example.com", + Bucket: "source", + Prefix: "incoming", + Region: DefaultS3Region, + ForcePath: &forcePathStyle, + }, + destination: Destination{ + Backend: BackendS3, + Endpoint: "https://s3.example.com", + Bucket: "destination", + Prefix: "archive", + Region: DefaultS3Region, + ForcePath: &forcePathStyle, + }, + }, + { + name: "s3 partial credentials", + source: Backend{ + Backend: BackendS3, + Endpoint: "https://s3.example.com", + Bucket: "source", + Region: DefaultS3Region, + Creds: Credentials{AccessKeyIDEnv: "ACCESS_KEY_ID"}, + }, + destination: Destination{ + Backend: BackendS3, + Endpoint: "https://s3.example.com", + Bucket: "destination", + Region: DefaultS3Region, + Creds: Credentials{AccessKeyIDEnv: "ACCESS_KEY_ID"}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sourceErrors := validateBackend(nil, "source", backendViewFromSource(tt.source)) + destinationErrors := validateBackend(nil, "destination", backendViewFromDestination(tt.destination)) + if got := len(sourceErrors) > 0; got != tt.wantErr { + t.Fatalf("source validation errors = %#v, wantErr %t", sourceErrors, tt.wantErr) + } + if got := len(destinationErrors) > 0; got != tt.wantErr { + t.Fatalf("destination validation errors = %#v, wantErr %t", destinationErrors, tt.wantErr) + } + }) + } +} diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 5cd25e4..00dcf4f 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -134,30 +134,24 @@ func duration(value Duration) *Duration { } func applyBackendDefaults(backend *Backend) { - if backend.Backend == BackendSSH { - if backend.Port == 0 { - backend.Port = 22 - } - if backend.SSH.HostKeyPolicy == "" { - backend.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew - } - } - if backend.Backend == BackendS3 { - applyS3Defaults(&backend.Region, &backend.Prefix, &backend.ForcePath) - } + applyStorageBackendDefaults(backend.Backend, &backend.Port, &backend.SSH, &backend.Region, &backend.Prefix, &backend.ForcePath) } func applyDestinationDefaults(destination *Destination) { - if destination.Backend == BackendSSH { - if destination.Port == 0 { - destination.Port = 22 + applyStorageBackendDefaults(destination.Backend, &destination.Port, &destination.SSH, &destination.Region, &destination.Prefix, &destination.ForcePath) +} + +func applyStorageBackendDefaults(backend string, port *int, ssh *SSH, region, prefix *string, forcePath **bool) { + if backend == BackendSSH { + if *port == 0 { + *port = 22 } - if destination.SSH.HostKeyPolicy == "" { - destination.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew + if ssh.HostKeyPolicy == "" { + ssh.HostKeyPolicy = HostKeyPolicyAcceptNew } } - if destination.Backend == BackendS3 { - applyS3Defaults(&destination.Region, &destination.Prefix, &destination.ForcePath) + if backend == BackendS3 { + applyS3Defaults(region, prefix, forcePath) } } diff --git a/internal/config/validate.go b/internal/config/validate.go index af93a24..102f099 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -100,7 +100,7 @@ func validateSourceBackend(errs ValidationErrors, context string, backend Backen if backend.Backend == BackendHTTPUpload { return validateHTTPUploadSource(errs, context, backend.Upload) } - return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.Endpoint, backend.Bucket, backend.Prefix, backend.SSH.HostKeyPolicy, backend.Creds) + return validateBackend(errs, context, backendViewFromSource(backend)) } func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors { @@ -108,7 +108,7 @@ func validateDestinationBackend(errs ValidationErrors, context string, destinati errs = append(errs, context+".backend "+BackendHTTPUpload+" is only supported for sources") return errs } - return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.Endpoint, destination.Bucket, destination.Prefix, destination.SSH.HostKeyPolicy, destination.Creds) + return validateBackend(errs, context, backendViewFromDestination(destination)) } func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTPUpload) ValidationErrors { @@ -124,47 +124,47 @@ func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTP return errs } -func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, endpoint, bucket, prefix string, hostKeyPolicy HostKeyPolicy, creds Credentials) ValidationErrors { - switch backend { +func validateBackend(errs ValidationErrors, context string, backend backendView) ValidationErrors { + switch backend.Backend { case "": errs = append(errs, context+".backend is required") case BackendLocal: - if path == "" { + if backend.Path == "" { errs = append(errs, context+".path is required for local backend") } case BackendSSH: - if host == "" { + if backend.Host == "" { errs = append(errs, context+".host is required for ssh backend") } - if path == "" { + if backend.Path == "" { errs = append(errs, context+".path is required for ssh backend") } - if port < 0 || port > 65535 { + if backend.Port < 0 || backend.Port > 65535 { errs = append(errs, context+".port must be between 1 and 65535") } - if port == 0 { + if backend.Port == 0 { errs = append(errs, context+".port is required for ssh backend after defaults are applied") } - if hostKeyPolicy != "" { - if _, ok := NormalizeHostKeyPolicy(string(hostKeyPolicy)); !ok { + if backend.SSH.HostKeyPolicy != "" { + if _, ok := NormalizeHostKeyPolicy(string(backend.SSH.HostKeyPolicy)); !ok { errs = append(errs, context+".host_key_policy must be strict, true, accept-new, off, or false") } } case BackendS3: - if endpoint == "" { + if backend.Endpoint == "" { errs = append(errs, context+".endpoint is required for s3 backend") } - if bucket == "" { + if backend.Bucket == "" { errs = append(errs, context+".bucket is required for s3 backend") } - if err := ValidateS3Prefix(prefix); err != nil { + if err := ValidateS3Prefix(backend.Prefix); err != nil { errs = append(errs, context+".prefix must be a clean relative slash-separated path") } - if (creds.AccessKeyIDEnv == "") != (creds.SecretAccessKeyEnv == "") { + if (backend.Creds.AccessKeyIDEnv == "") != (backend.Creds.SecretAccessKeyEnv == "") { errs = append(errs, context+".credentials.access_key_id_env and credentials.secret_access_key_env must be configured together") } default: - errs = append(errs, context+".backend "+backend+" is unsupported") + errs = append(errs, context+".backend "+backend.Backend+" is unsupported") } return errs }