eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add SSH SFTP backend support

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-31 16:53:37 +00:00
parent 1ad566264f
commit 84f77ec0d0
29 changed files with 1629 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/config/config.go
View File

@@ -14,6 +14,9 @@ type Pipeline struct {
type Destination struct {
ID string `yaml:"id"`
Backend string `yaml:"backend"`
Host string `yaml:"host"`
User string `yaml:"user"`
Port int `yaml:"port"`
Path string `yaml:"path"`
URI string `yaml:"uri"`
Endpoint string `yaml:"endpoint"`
@@ -22,6 +25,7 @@ type Destination struct {
Region string `yaml:"region"`
ForcePath bool `yaml:"force_path_style"`
Creds Credentials `yaml:"credentials"`
SSH SSH `yaml:",inline"`
Publish *PublishPolicy `yaml:"publish"`
Transform Transform `yaml:"transform"`
Transfer TransferPolicy `yaml:"transfer"`
@@ -29,6 +33,9 @@ type Destination struct {
type Backend struct {
Backend string `yaml:"backend"`
Host string `yaml:"host"`
User string `yaml:"user"`
Port int `yaml:"port"`
Path string `yaml:"path"`
URI string `yaml:"uri"`
Endpoint string `yaml:"endpoint"`
@@ -37,6 +44,13 @@ type Backend struct {
Region string `yaml:"region"`
ForcePath bool `yaml:"force_path_style"`
Creds Credentials `yaml:"credentials"`
SSH SSH `yaml:",inline"`
}
type SSH struct {
KeyFile string `yaml:"ssh_key_file"`
KnownHosts string `yaml:"known_hosts"`
HostKeyPolicy HostKeyPolicy `yaml:"host_key_policy"`
}
type Credentials struct {

24
internal/config/defaults.go
View File

@@ -25,11 +25,13 @@ const (
func ApplyDefaults(cfg *Config) {
for pipelineIndex := range cfg.Pipelines {
pipeline := &cfg.Pipelines[pipelineIndex]
applyBackendDefaults(&pipeline.Source)
if pipeline.Validation.OnDigestMismatch == "" {
pipeline.Validation.OnDigestMismatch = ValidationActionFail
}
for destinationIndex := range pipeline.Destinations {
destination := &pipeline.Destinations[destinationIndex]
applyDestinationDefaults(destination)
if destination.Publish == nil {
destination.Publish = &PublishPolicy{Source: true}
}
@@ -48,3 +50,25 @@ func ApplyDefaults(cfg *Config) {
}
}
}
func applyBackendDefaults(backend *Backend) {
if backend.Backend == BackendSSH {
if backend.Port == 0 {
backend.Port = 22
}
if backend.SSH.HostKeyPolicy == "" {
backend.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew
}
}
}
func applyDestinationDefaults(destination *Destination) {
if destination.Backend == BackendSSH {
if destination.Port == 0 {
destination.Port = 22
}
if destination.SSH.HostKeyPolicy == "" {
destination.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew
}
}
}

99
internal/config/load_test.go
View File

@@ -53,7 +53,9 @@ pipelines:
html: false
- id: static-site
backend: ssh
uri: ssh://deploy@example.com:22
host: example.com
user: deploy
port: 22
path: /srv/www/reports
publish:
source: false
@@ -87,13 +89,19 @@ pipelines:
- id: ssh-backend
source:
backend: ssh
uri: ssh://reports@example.com:22
host: source.example.com
user: reports
path: /source
destinations:
- id: ssh-destination
backend: ssh
uri: ssh://deploy@example.com:22
host: destination.example.com
user: deploy
port: 2222
path: /destination
ssh_key_file: /home/deploy/.ssh/id_ed25519
known_hosts: /home/deploy/.ssh/known_hosts
host_key_policy: strict
`,
"s3": `
pipelines:
@@ -171,7 +179,7 @@ func TestLoadFileRejectsMissingRequiredFields(t *testing.T) {
"destinations": `pipelines: [{id: reports, source: {backend: local, path: /source}}]`,
"destination id": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{backend: local, path: /archive}]}]`,
"local path": `pipelines: [{id: reports, source: {backend: local}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"ssh uri": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"ssh host": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"s3 bucket": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com"}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"publish outputs": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, publish: {source: false, html: false}}]}]`,
}
@@ -183,6 +191,88 @@ func TestLoadFileRejectsMissingRequiredFields(t *testing.T) {
}
}
func TestLoadFileDefaultsSSHConfig(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: ssh-defaults
source:
backend: ssh
host: source.example.com
path: /source
destinations:
- id: archive
backend: ssh
host: destination.example.com
path: /archive
host_key_policy: false
`)
source := cfg.Pipelines[0].Source
if source.Port != 22 {
t.Fatalf("source port = %d, want 22", source.Port)
}
if source.SSH.HostKeyPolicy != HostKeyPolicyAcceptNew {
t.Fatalf("source host key policy = %q, want accept-new", source.SSH.HostKeyPolicy)
}
destination := cfg.Pipelines[0].Destinations[0]
if destination.Port != 22 {
t.Fatalf("destination port = %d, want 22", destination.Port)
}
if destination.SSH.HostKeyPolicy != HostKeyPolicyOff {
t.Fatalf("destination host key policy = %q, want off", destination.SSH.HostKeyPolicy)
}
}
func TestLoadFileNormalizesSSHHostKeyPolicies(t *testing.T) {
tests := map[string]HostKeyPolicy{
`true`: HostKeyPolicyStrict,
`"true"`: HostKeyPolicyStrict,
`strict`: HostKeyPolicyStrict,
`accept-new`: HostKeyPolicyAcceptNew,
`false`: HostKeyPolicyOff,
`"false"`: HostKeyPolicyOff,
`off`: HostKeyPolicyOff,
`"STRICT"`: HostKeyPolicyStrict,
`"ACCEPT-NEW"`: HostKeyPolicyAcceptNew,
`"OFF"`: HostKeyPolicyOff,
}
for value, want := range tests {
t.Run(value, func(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: ssh-policy
source:
backend: ssh
host: source.example.com
path: /source
host_key_policy: `+value+`
destinations:
- id: archive
backend: local
path: /archive
`)
if got := cfg.Pipelines[0].Source.SSH.HostKeyPolicy; got != want {
t.Fatalf("host key policy = %q, want %q", got, want)
}
})
}
}
func TestLoadFileRejectsSSHURIExecutionConfig(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: ssh
uri: ssh://reports@example.com:22
path: /source
destinations:
- id: archive
backend: local
path: /archive
`, "uri is not supported for ssh backend")
}
func TestLoadFileRejectsUnsupportedBackend(t *testing.T) {
assertLoadError(t, `
pipelines:
@@ -267,6 +357,7 @@ func TestExampleConfigsLoad(t *testing.T) {
"../../examples/local-publish.yml",
"../../examples/local-html.yml",
"../../examples/fan-out.yml",
"../../examples/ssh-destination.yml",
} {
t.Run(path, func(t *testing.T) {
if _, err := LoadFile(path); err != nil {

64
internal/config/ssh.go Normal file
View File

@@ -0,0 +1,64 @@
package config
import (
"fmt"
"strings"
"gopkg.in/yaml.v3"
)
type HostKeyPolicy string
const (
HostKeyPolicyStrict HostKeyPolicy = "strict"
HostKeyPolicyAcceptNew HostKeyPolicy = "accept-new"
HostKeyPolicyOff HostKeyPolicy = "off"
)
func (p *HostKeyPolicy) UnmarshalYAML(value *yaml.Node) error {
switch value.Kind {
case yaml.ScalarNode:
default:
return fmt.Errorf("host_key_policy must be a boolean or string")
}
switch value.Tag {
case "!!bool":
var enabled bool
if err := value.Decode(&enabled); err != nil {
return err
}
if enabled {
*p = HostKeyPolicyStrict
} else {
*p = HostKeyPolicyOff
}
return nil
case "!!str":
var raw string
if err := value.Decode(&raw); err != nil {
return err
}
normalized, ok := NormalizeHostKeyPolicy(raw)
if !ok {
return fmt.Errorf("host_key_policy must be strict, true, accept-new, off, or false")
}
*p = normalized
return nil
default:
return fmt.Errorf("host_key_policy must be a boolean or string")
}
}
func NormalizeHostKeyPolicy(value string) (HostKeyPolicy, bool) {
switch strings.ToLower(value) {
case "", string(HostKeyPolicyAcceptNew):
return HostKeyPolicyAcceptNew, true
case string(HostKeyPolicyStrict), "true":
return HostKeyPolicyStrict, true
case string(HostKeyPolicyOff), "false":
return HostKeyPolicyOff, true
default:
return "", false
}
}

32
internal/config/validate.go
View File

@@ -37,7 +37,7 @@ func Validate(cfg Config) error {
pipelineIDs[pipeline.ID] = struct{}{}
}
errs = validateBackend(errs, pipelineContext+".source", pipeline.Source.Backend, pipeline.Source.Path, pipeline.Source.URI, pipeline.Source.Endpoint, pipeline.Source.Bucket)
errs = validateSourceBackend(errs, pipelineContext+".source", pipeline.Source)
errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation)
if len(pipeline.Destinations) == 0 {
errs = append(errs, pipelineContext+".destinations is required")
@@ -56,7 +56,7 @@ func Validate(cfg Config) error {
destinationIDs[destination.ID] = struct{}{}
}
errs = validateBackend(errs, destinationContext, destination.Backend, destination.Path, destination.URI, destination.Endpoint, destination.Bucket)
errs = validateDestinationBackend(errs, destinationContext, destination)
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
}
@@ -68,7 +68,15 @@ func Validate(cfg Config) error {
return nil
}
func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoint, bucket string) ValidationErrors {
func validateSourceBackend(errs ValidationErrors, context string, backend Backend) ValidationErrors {
return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.URI, backend.Endpoint, backend.Bucket, backend.SSH.HostKeyPolicy)
}
func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors {
return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.URI, destination.Endpoint, destination.Bucket, destination.SSH.HostKeyPolicy)
}
func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, uri, endpoint, bucket string, hostKeyPolicy HostKeyPolicy) ValidationErrors {
switch backend {
case "":
errs = append(errs, context+".backend is required")
@@ -77,12 +85,26 @@ func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoin
errs = append(errs, context+".path is required for local backend")
}
case BackendSSH:
if uri == "" {
errs = append(errs, context+".uri is required for ssh backend")
if host == "" {
errs = append(errs, context+".host is required for ssh backend")
}
if path == "" {
errs = append(errs, context+".path is required for ssh backend")
}
if uri != "" {
errs = append(errs, context+".uri is not supported for ssh backend; use host, user, port, and path")
}
if port < 0 || port > 65535 {
errs = append(errs, context+".port must be between 1 and 65535")
}
if port == 0 {
errs = append(errs, context+".port is required for ssh backend after defaults are applied")
}
if hostKeyPolicy != "" {
if _, ok := NormalizeHostKeyPolicy(string(hostKeyPolicy)); !ok {
errs = append(errs, context+".host_key_policy must be strict, true, accept-new, off, or false")
}
}
case BackendS3:
if endpoint == "" {
errs = append(errs, context+".endpoint is required for s3 backend")