Add SSH SFTP backend support

This commit is contained in:
2026-05-31 16:53:37 +00:00
parent 1ad566264f
commit 84f77ec0d0
29 changed files with 1629 additions and 40 deletions

View File

@@ -0,0 +1,61 @@
package ssh
import (
"fmt"
"io"
"net"
"os"
cryptossh "golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
)
type agentDialer func(network, address string) (net.Conn, error)
func authMethods(keyFile string) ([]cryptossh.AuthMethod, func(), error) {
return authMethodsWithAgent(os.Getenv("SSH_AUTH_SOCK"), net.Dial, keyFile)
}
func authMethodsWithAgent(agentSocket string, dial agentDialer, keyFile string) ([]cryptossh.AuthMethod, func(), error) {
var methods []cryptossh.AuthMethod
var closers []io.Closer
if agentSocket != "" {
methods = append(methods, cryptossh.PublicKeysCallback(func() ([]cryptossh.Signer, error) {
conn, err := dial("unix", agentSocket)
if err != nil {
return nil, err
}
closers = append(closers, conn)
return agent.NewClient(conn).Signers()
}))
}
if keyFile != "" {
signer, err := signerFromKeyFile(keyFile)
if err != nil {
return nil, nil, err
}
methods = append(methods, cryptossh.PublicKeys(signer))
}
if len(methods) == 0 {
return nil, nil, fmt.Errorf("no SSH auth methods configured; set SSH_AUTH_SOCK or ssh_key_file")
}
return methods, func() { closeAll(closers) }, nil
}
func signerFromKeyFile(path string) (cryptossh.Signer, error) {
key, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read ssh_key_file %q: %w", path, err)
}
signer, err := cryptossh.ParsePrivateKey(key)
if err != nil {
return nil, fmt.Errorf("parse ssh_key_file %q: %w", path, err)
}
return signer, nil
}
func closeAll(closers []io.Closer) {
for _, closer := range closers {
_ = closer.Close()
}
}

View File

@@ -0,0 +1,59 @@
package ssh
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"os"
"path/filepath"
"testing"
)
func TestAuthMethodsPreferAgentBeforeKeyFile(t *testing.T) {
keyFile := writePrivateKey(t)
methods, cleanup, err := authMethodsWithAgent("/tmp/ssh-agent.sock", nil, keyFile)
if err != nil {
t.Fatalf("authMethodsWithAgent() error = %v", err)
}
defer cleanup()
if got, want := len(methods), 2; got != want {
t.Fatalf("auth method count = %d, want %d", got, want)
}
}
func TestAuthMethodsLoadsKeyFile(t *testing.T) {
keyFile := writePrivateKey(t)
methods, cleanup, err := authMethodsWithAgent("", nil, keyFile)
if err != nil {
t.Fatalf("authMethodsWithAgent() error = %v", err)
}
defer cleanup()
if got, want := len(methods), 1; got != want {
t.Fatalf("auth method count = %d, want %d", got, want)
}
}
func TestAuthMethodsRejectsMissingAuth(t *testing.T) {
_, _, err := authMethodsWithAgent("", nil, "")
if err == nil {
t.Fatal("authMethodsWithAgent() error = nil, want error")
}
}
func writePrivateKey(t *testing.T) string {
t.Helper()
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate key: %v", err)
}
data := pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
})
path := filepath.Join(t.TempDir(), "id_rsa")
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatalf("write private key: %v", err)
}
return path
}

View File

@@ -0,0 +1,445 @@
package ssh
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"github.com/pkg/sftp"
cryptossh "golang.org/x/crypto/ssh"
)
type Backend struct {
client *sftp.Client
sshClient *cryptossh.Client
root string
}
func New(ctx context.Context, options Options) (*Backend, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
options, err := options.normalized()
if err != nil {
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Root, storage.ErrInvalidPath, err)
}
hostKeyCallback, err := hostKeyCallback(options)
if err != nil {
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.KnownHosts, storage.ErrInvalidPath, err)
}
auth, cleanupAuth, err := authMethods(options.KeyFile)
if err != nil {
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.KeyFile, storage.ErrPermission, err)
}
defer cleanupAuth()
sshClient, err := cryptossh.Dial("tcp", options.address(), &cryptossh.ClientConfig{
User: options.User,
Auth: auth,
HostKeyCallback: hostKeyCallback,
Timeout: 30 * time.Second,
})
if err != nil {
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.address(), storage.ErrUnknown, err)
}
client, err := sftp.NewClient(sshClient)
if err != nil {
_ = sshClient.Close()
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.address(), storage.ErrUnknown, err)
}
return &Backend{client: client, sshClient: sshClient, root: options.Root}, nil
}
func (b *Backend) Close() error {
var err error
if b.client != nil {
err = b.client.Close()
}
if b.sshClient != nil {
if closeErr := b.sshClient.Close(); err == nil {
err = closeErr
}
}
return err
}
func (b *Backend) ReadFile(ctx context.Context, logicalPath string) ([]byte, error) {
reader, err := b.OpenReader(ctx, logicalPath)
if err != nil {
return nil, err
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil {
return nil, storage.NewError(storage.OpReadFile, BackendName, logicalPath, storage.ErrUnknown, err)
}
return data, nil
}
func (b *Backend) OpenReader(ctx context.Context, logicalPath string) (io.ReadCloser, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
nativePath, err := b.nativePath(logicalPath, false)
if err != nil {
return nil, err
}
if err := b.rejectSymlinkAncestors(ctx, logicalPath, true); err != nil {
return nil, err
}
info, err := b.client.Lstat(nativePath)
if err != nil {
return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
}
if !info.Mode().IsRegular() {
return nil, storage.NewError(storage.OpOpenReader, BackendName, logicalPath, storage.ErrUnsupported, nil)
}
file, err := b.client.Open(nativePath)
if err != nil {
return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
}
return file, nil
}
func (b *Backend) WriteFile(ctx context.Context, logicalPath string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
opts.Size = int64(len(data))
opts.SizeKnown = true
return b.WriteFrom(ctx, logicalPath, bytes.NewReader(data), opts)
}
func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
if err := ctx.Err(); err != nil {
return storage.Entry{}, err
}
nativePath, err := b.nativePath(logicalPath, false)
if err != nil {
return storage.Entry{}, err
}
if err := b.rejectSymlinkAncestors(ctx, parentOf(logicalPath), true); err != nil {
return storage.Entry{}, err
}
if info, err := b.client.Lstat(nativePath); err == nil {
if !opts.Overwrite {
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrAlreadyExist, nil)
}
if !info.Mode().IsRegular() {
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, nil)
}
} else if !isNotExist(err) {
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
}
parentNative := path.Dir(nativePath)
if err := b.client.MkdirAll(parentNative); err != nil {
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
}
writePath := nativePath
if opts.PreferAtomic {
writePath = path.Join(parentNative, fmt.Sprintf(".distributor-write-%d", time.Now().UnixNano()))
}
file, err := b.client.Create(writePath)
if err != nil {
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
}
cleanup := opts.PreferAtomic
defer func() {
if cleanup {
_ = b.client.Remove(writePath)
}
}()
written, copyErr := io.Copy(file, r)
closeErr := file.Close()
if copyErr != nil {
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, copyErr)
}
if closeErr != nil {
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, closeErr)
}
if opts.SizeKnown && written != opts.Size {
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", written, opts.Size))
}
if opts.PreferAtomic {
if err := b.client.Rename(writePath, nativePath); err != nil {
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
}
cleanup = false
}
return b.Stat(ctx, logicalPath)
}
func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) {
if err := ctx.Err(); err != nil {
return storage.Entry{}, err
}
nativePath, err := b.nativePath(logicalPath, true)
if err != nil {
return storage.Entry{}, err
}
info, err := b.client.Lstat(nativePath)
if err != nil {
return storage.Entry{}, b.translateError(storage.OpStat, logicalPath, err)
}
return entryFromInfo(logicalPath, info), nil
}
func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
if err := ctx.Err(); err != nil {
return err
}
nativePrefix, err := b.nativePath(prefix, true)
if err != nil {
return err
}
info, err := b.client.Lstat(nativePrefix)
if err != nil {
if isNotExist(err) {
return nil
}
return b.translateError(storage.OpWalk, prefix, err)
}
visited := 0
emit := func(entry storage.Entry) error {
if err := ctx.Err(); err != nil {
return err
}
if opts.Limit > 0 && visited >= opts.Limit {
return storage.ErrStopWalk
}
visited++
if err := fn(entry); err != nil {
if errors.Is(err, storage.ErrStopWalk) {
return storage.ErrStopWalk
}
return storage.NewError(storage.OpWalk, BackendName, entry.Path, storage.ErrUnknown, err)
}
return nil
}
if !info.IsDir() {
if err := emit(entryFromInfo(prefix, info)); errors.Is(err, storage.ErrStopWalk) {
return nil
} else if err != nil {
return err
}
return nil
}
if err := b.walkDirectory(ctx, prefix, nativePrefix, opts, emit); errors.Is(err, storage.ErrStopWalk) {
return nil
} else if err != nil {
return err
}
return nil
}
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
found := false
err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
found = true
return storage.ErrStopWalk
})
if err != nil {
return false, err
}
return found, nil
}
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
if err := ctx.Err(); err != nil {
return err
}
targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
if err != nil {
return err
}
for _, target := range targets {
nativePath, err := b.nativePath(target, false)
if err != nil {
return err
}
if nativePath == b.root {
return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrInvalidPath, nil)
}
info, err := b.client.Lstat(nativePath)
if err != nil {
if opts.IgnoreMissing && isNotExist(err) {
continue
}
return b.translateError(storage.OpDeleteManagedBundle, target, err)
}
if info.IsDir() {
return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrUnsupported, nil)
}
if err := b.client.Remove(nativePath); err != nil {
return b.translateError(storage.OpDeleteManagedBundle, target, err)
}
if opts.PruneEmptyDirs {
b.pruneEmptyParents(parentOf(target))
}
}
return nil
}
func (b *Backend) walkDirectory(ctx context.Context, logicalPrefix, nativePrefix string, opts storage.WalkOptions, emit func(storage.Entry) error) error {
entries, err := b.client.ReadDir(nativePrefix)
if err != nil {
return b.translateError(storage.OpWalk, logicalPrefix, err)
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
for _, info := range entries {
if err := ctx.Err(); err != nil {
return err
}
logicalPath := info.Name()
if logicalPrefix != "" {
logicalPath = logicalPrefix + "/" + info.Name()
}
if err := emit(entryFromInfo(logicalPath, info)); err != nil {
return err
}
if opts.Recursive && info.IsDir() {
if err := b.walkDirectory(ctx, logicalPath, path.Join(nativePrefix, info.Name()), opts, emit); err != nil {
return err
}
}
}
return nil
}
func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) {
if logicalPath == "" {
if !allowEmpty {
return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
}
return b.root, nil
}
if err := storage.ValidatePath(logicalPath); err != nil {
return "", err
}
nativePath := path.Clean(path.Join(b.root, logicalPath))
if !withinRoot(b.root, nativePath) {
return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
}
return nativePath, nil
}
func (b *Backend) rejectSymlinkAncestors(ctx context.Context, logicalPath string, includeFinal bool) error {
if logicalPath == "" {
return nil
}
if err := storage.ValidatePath(logicalPath); err != nil {
return err
}
segments := strings.Split(logicalPath, "/")
limit := len(segments)
if !includeFinal {
limit--
}
current := ""
for index := 0; index < limit; index++ {
if err := ctx.Err(); err != nil {
return err
}
if current == "" {
current = segments[index]
} else {
current += "/" + segments[index]
}
nativePath, err := b.nativePath(current, false)
if err != nil {
return err
}
info, err := b.client.Lstat(nativePath)
if err != nil {
if isNotExist(err) {
return nil
}
return b.translateError(storage.OpStat, current, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return storage.NewError(storage.OpStat, BackendName, current, storage.ErrUnsupported, nil)
}
}
return nil
}
func (b *Backend) pruneEmptyParents(logicalPath string) {
for logicalPath != "" {
nativePath, err := b.nativePath(logicalPath, false)
if err != nil || nativePath == b.root {
return
}
if err := b.client.RemoveDirectory(nativePath); err != nil {
return
}
logicalPath = parentOf(logicalPath)
}
}
func withinRoot(root, candidate string) bool {
if candidate == root {
return true
}
if root == "/" {
return strings.HasPrefix(candidate, "/")
}
return strings.HasPrefix(candidate, strings.TrimSuffix(root, "/")+"/")
}
func parentOf(logicalPath string) string {
index := strings.LastIndex(logicalPath, "/")
if index == -1 {
return ""
}
return logicalPath[:index]
}
func isNotExist(err error) bool {
return errors.Is(err, fs.ErrNotExist) || errors.Is(err, os.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile)
}
func (b *Backend) translateError(op, logicalPath string, err error) error {
kind := storage.ErrUnknown
switch {
case isNotExist(err):
kind = storage.ErrNotFound
case errors.Is(err, fs.ErrExist), errors.Is(err, os.ErrExist):
kind = storage.ErrAlreadyExist
case errors.Is(err, fs.ErrPermission), errors.Is(err, os.ErrPermission), errors.Is(err, sftp.ErrSSHFxPermissionDenied):
kind = storage.ErrPermission
case errors.Is(err, sftp.ErrSSHFxOpUnsupported):
kind = storage.ErrUnsupported
case errors.Is(err, sftp.ErrSSHFxNoConnection), errors.Is(err, sftp.ErrSSHFxConnectionLost):
kind = storage.ErrTemporary
}
return storage.NewError(op, BackendName, logicalPath, kind, err)
}
func entryFromInfo(logicalPath string, info fs.FileInfo) storage.Entry {
entryType := storage.EntryTypeOther
switch {
case info.Mode()&os.ModeSymlink != 0:
entryType = storage.EntryTypeSymlink
case info.Mode().IsRegular():
entryType = storage.EntryTypeFile
case info.IsDir():
entryType = storage.EntryTypeDirectory
}
return storage.Entry{
Path: logicalPath,
Type: entryType,
Size: info.Size(),
}
}

View File

@@ -0,0 +1,81 @@
package ssh
import (
"errors"
"fmt"
"net"
"os"
cryptossh "golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
func hostKeyCallback(options Options) (cryptossh.HostKeyCallback, error) {
switch options.HostKeyPolicy {
case HostKeyPolicyOff:
return cryptossh.InsecureIgnoreHostKey(), nil
case HostKeyPolicyStrict:
if options.KnownHosts == "" {
return nil, fmt.Errorf("known_hosts is required for strict host key checking")
}
callback, err := knownhosts.New(options.KnownHosts)
if err != nil {
return nil, fmt.Errorf("load known_hosts %q: %w", options.KnownHosts, err)
}
return callback, nil
case HostKeyPolicyAcceptNew:
return acceptNewHostKeyCallback(options)
default:
return nil, fmt.Errorf("host_key_policy must be strict, accept-new, or off")
}
}
func acceptNewHostKeyCallback(options Options) (cryptossh.HostKeyCallback, error) {
var checker cryptossh.HostKeyCallback
if options.KnownHosts != "" {
loaded, err := knownhosts.New(options.KnownHosts)
if err == nil {
checker = loaded
} else if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("load known_hosts %q: %w", options.KnownHosts, err)
}
}
return func(hostname string, remote net.Addr, key cryptossh.PublicKey) error {
if checker != nil {
err := checker(hostname, remote, key)
if err == nil {
return nil
}
var keyErr *knownhosts.KeyError
if !errors.As(err, &keyErr) {
return err
}
if len(keyErr.Want) > 0 {
return fmt.Errorf("host key for %s has changed: %w", hostname, err)
}
}
if options.KnownHosts == "" {
return fmt.Errorf("host key for %s is unknown and no writable known_hosts path is available", hostname)
}
if err := appendKnownHost(options.KnownHosts, hostname, key); err != nil {
return err
}
loaded, err := knownhosts.New(options.KnownHosts)
if err == nil {
checker = loaded
}
return nil
}, nil
}
func appendKnownHost(path, host string, key cryptossh.PublicKey) error {
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return fmt.Errorf("persist accepted host key to known_hosts %q: %w", path, err)
}
defer file.Close()
if _, err := fmt.Fprintln(file, knownhosts.Line([]string{knownhosts.Normalize(host)}, key)); err != nil {
return fmt.Errorf("persist accepted host key to known_hosts %q: %w", path, err)
}
return nil
}

View File

@@ -0,0 +1,87 @@
package ssh
import (
"crypto/rand"
"crypto/rsa"
"net"
"os"
"path/filepath"
"strings"
"testing"
cryptossh "golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
func TestAcceptNewHostKeyCallbackPersistsUnknownHost(t *testing.T) {
key := testPublicKey(t)
knownHosts := filepath.Join(t.TempDir(), "known_hosts")
callback, err := acceptNewHostKeyCallback(Options{KnownHosts: knownHosts})
if err != nil {
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
}
if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil {
t.Fatalf("callback() error = %v", err)
}
data, err := os.ReadFile(knownHosts)
if err != nil {
t.Fatalf("read known_hosts: %v", err)
}
if !strings.Contains(string(data), "example.com") {
t.Fatalf("known_hosts = %q, want example.com entry", data)
}
if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil {
t.Fatalf("second callback() error = %v", err)
}
}
func TestAcceptNewHostKeyCallbackRejectsChangedHostKey(t *testing.T) {
first := testPublicKey(t)
second := testPublicKey(t)
knownHosts := filepath.Join(t.TempDir(), "known_hosts")
if err := os.WriteFile(knownHosts, []byte(knownhosts.Line([]string{knownhosts.Normalize("example.com:22")}, first)+"\n"), 0o600); err != nil {
t.Fatalf("write known_hosts: %v", err)
}
callback, err := acceptNewHostKeyCallback(Options{KnownHosts: knownHosts})
if err != nil {
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
}
err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, second)
if err == nil || !strings.Contains(err.Error(), "has changed") {
t.Fatalf("callback() error = %v, want changed host key", err)
}
}
func TestAcceptNewHostKeyCallbackRequiresWritableKnownHostsForUnknownHost(t *testing.T) {
callback, err := acceptNewHostKeyCallback(Options{})
if err != nil {
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
}
err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, testPublicKey(t))
if err == nil || !strings.Contains(err.Error(), "no writable known_hosts path") {
t.Fatalf("callback() error = %v, want no writable known_hosts path", err)
}
}
func TestStrictHostKeyCallbackRequiresKnownHosts(t *testing.T) {
_, err := hostKeyCallback(Options{HostKeyPolicy: HostKeyPolicyStrict})
if err == nil || !strings.Contains(err.Error(), "known_hosts is required") {
t.Fatalf("hostKeyCallback() error = %v, want known_hosts required", err)
}
}
func testPublicKey(t *testing.T) cryptossh.PublicKey {
t.Helper()
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("generate key: %v", err)
}
publicKey, err := cryptossh.NewPublicKey(&privateKey.PublicKey)
if err != nil {
t.Fatalf("new public key: %v", err)
}
return publicKey
}

View File

@@ -0,0 +1,39 @@
package ssh
import (
"context"
"os"
"strconv"
"testing"
)
func TestIntegrationSSHBackendStatRoot(t *testing.T) {
host := os.Getenv("DISTRIBUTOR_TEST_SSH_HOST")
if host == "" {
t.Skip("DISTRIBUTOR_TEST_SSH_HOST is not set")
}
port := 22
if raw := os.Getenv("DISTRIBUTOR_TEST_SSH_PORT"); raw != "" {
parsed, err := strconv.Atoi(raw)
if err != nil {
t.Fatalf("parse DISTRIBUTOR_TEST_SSH_PORT: %v", err)
}
port = parsed
}
backend, err := New(context.Background(), Options{
Host: host,
User: os.Getenv("DISTRIBUTOR_TEST_SSH_USER"),
Port: port,
Root: os.Getenv("DISTRIBUTOR_TEST_SSH_PATH"),
KeyFile: os.Getenv("DISTRIBUTOR_TEST_SSH_KEY_FILE"),
KnownHosts: os.Getenv("DISTRIBUTOR_TEST_SSH_KNOWN_HOSTS"),
HostKeyPolicy: HostKeyPolicyStrict,
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
defer backend.Close()
if _, err := backend.Stat(context.Background(), ""); err != nil {
t.Fatalf("Stat(root) error = %v", err)
}
}

View File

@@ -0,0 +1,77 @@
package ssh
import (
"fmt"
"os"
"os/user"
"path"
"path/filepath"
"strconv"
)
const (
BackendName = "ssh"
HostKeyPolicyStrict HostKeyPolicy = "strict"
HostKeyPolicyAcceptNew HostKeyPolicy = "accept-new"
HostKeyPolicyOff HostKeyPolicy = "off"
)
type HostKeyPolicy string
type Options struct {
Host string
User string
Port int
Root string
KeyFile string
KnownHosts string
HostKeyPolicy HostKeyPolicy
}
func (o Options) normalized() (Options, error) {
if o.Host == "" {
return Options{}, fmt.Errorf("host is required")
}
if o.User == "" {
current, err := user.Current()
if err != nil || current.Username == "" {
return Options{}, fmt.Errorf("user is required when current OS user cannot be determined")
}
o.User = current.Username
}
if o.Port == 0 {
o.Port = 22
}
if o.Port < 1 || o.Port > 65535 {
return Options{}, fmt.Errorf("port must be between 1 and 65535")
}
if o.Root == "" {
return Options{}, fmt.Errorf("path is required")
}
o.Root = path.Clean(o.Root)
if o.HostKeyPolicy == "" {
o.HostKeyPolicy = HostKeyPolicyAcceptNew
}
switch o.HostKeyPolicy {
case HostKeyPolicyStrict, HostKeyPolicyAcceptNew, HostKeyPolicyOff:
default:
return Options{}, fmt.Errorf("host_key_policy must be strict, accept-new, or off")
}
if o.KnownHosts == "" && o.HostKeyPolicy != HostKeyPolicyOff {
o.KnownHosts = defaultKnownHostsPath()
}
return o, nil
}
func (o Options) address() string {
return o.Host + ":" + strconv.Itoa(o.Port)
}
func defaultKnownHostsPath() string {
home, err := os.UserHomeDir()
if err != nil || home == "" {
return ""
}
return filepath.Join(home, ".ssh", "known_hosts")
}

View File

@@ -0,0 +1,118 @@
package ssh
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"github.com/pkg/sftp"
)
func TestOptionsNormalizeDefaultsUserPortAndHostKeyPolicy(t *testing.T) {
options, err := (Options{
Host: "example.com",
Root: "/reports",
}).normalized()
if err != nil {
t.Fatalf("normalized() error = %v", err)
}
if options.User == "" {
t.Fatal("normalized user is empty")
}
if options.Port != 22 {
t.Fatalf("port = %d, want 22", options.Port)
}
if options.HostKeyPolicy != HostKeyPolicyAcceptNew {
t.Fatalf("host key policy = %q, want accept-new", options.HostKeyPolicy)
}
}
func TestOptionsNormalizeRejectsInvalidFields(t *testing.T) {
tests := map[string]Options{
"host": {Root: "/reports"},
"port": {
Host: "example.com",
Port: 70000,
Root: "/reports",
},
"path": {
Host: "example.com",
},
"host key policy": {
Host: "example.com",
Root: "/reports",
HostKeyPolicy: "prompt",
},
}
for name, options := range tests {
t.Run(name, func(t *testing.T) {
if _, err := options.normalized(); err == nil {
t.Fatal("normalized() error = nil, want error")
}
})
}
}
func TestNativePathEnforcesLogicalPathRules(t *testing.T) {
backend := &Backend{root: "/srv/reports"}
tests := map[string]string{
"bundle/report.md": "/srv/reports/bundle/report.md",
"": "/srv/reports",
}
for logicalPath, want := range tests {
t.Run(logicalPath, func(t *testing.T) {
got, err := backend.nativePath(logicalPath, true)
if err != nil {
t.Fatalf("nativePath() error = %v", err)
}
if got != want {
t.Fatalf("nativePath() = %q, want %q", got, want)
}
})
}
for _, logicalPath := range []string{"/absolute", "../escape", "a/../b", `a\b`} {
t.Run("reject "+logicalPath, func(t *testing.T) {
_, err := backend.nativePath(logicalPath, true)
if err == nil || !storage.IsInvalidPath(err) {
t.Fatalf("nativePath() error = %v, want invalid path", err)
}
})
}
}
func TestNewRejectsMissingAuthBeforeDial(t *testing.T) {
t.Setenv("SSH_AUTH_SOCK", "")
_, err := New(context.Background(), Options{
Host: "example.com",
User: "reports",
Root: "/reports",
HostKeyPolicy: HostKeyPolicyOff,
})
if err == nil || !strings.Contains(err.Error(), "no SSH auth methods configured") {
t.Fatalf("New() error = %v, want missing auth", err)
}
}
func TestTranslateErrorMapsSFTPStatusCodes(t *testing.T) {
backend := &Backend{}
tests := []struct {
name string
err error
want storage.ErrorKind
}{
{name: "not found", err: sftp.ErrSSHFxNoSuchFile, want: storage.ErrNotFound},
{name: "permission", err: sftp.ErrSSHFxPermissionDenied, want: storage.ErrPermission},
{name: "unsupported", err: sftp.ErrSSHFxOpUnsupported, want: storage.ErrUnsupported},
{name: "temporary", err: sftp.ErrSSHFxConnectionLost, want: storage.ErrTemporary},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := backend.translateError(storage.OpStat, "report.md", tt.err)
if !storage.IsKind(err, tt.want) {
t.Fatalf("translateError() = %v, want kind %s", err, tt.want)
}
})
}
}

View File

@@ -3,14 +3,25 @@ package app
import (
"context"
"fmt"
"strconv"
"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
sshadapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/ssh"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
)
const storagePathKey = "path"
const (
sshHostKey = "host"
sshUserKey = "user"
sshPortKey = "port"
sshKeyFileKey = "ssh_key_file"
sshKnownHostsKey = "known_hosts"
sshHostKeyPolicyKey = "host_key_policy"
)
type backendFactory struct {
registry *storage.Registry
}
@@ -23,23 +34,64 @@ func newBackendFactory() *backendFactory {
}
return local.New(cfg[storagePathKey])
})
_ = registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
port, err := strconv.Atoi(cfg[sshPortKey])
if err != nil {
return nil, fmt.Errorf("ssh port: %w", err)
}
return sshadapter.New(ctx, sshadapter.Options{
Host: cfg[sshHostKey],
User: cfg[sshUserKey],
Port: port,
Root: cfg[storagePathKey],
KeyFile: cfg[sshKeyFileKey],
KnownHosts: cfg[sshKnownHostsKey],
HostKeyPolicy: sshadapter.HostKeyPolicy(cfg[sshHostKeyPolicyKey]),
})
})
return &backendFactory{registry: registry}
}
func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) {
if source.Backend != config.BackendLocal {
if source.Backend != config.BackendLocal && source.Backend != config.BackendSSH {
return nil, fmt.Errorf("source backend %s is not implemented for execution", source.Backend)
}
return f.registry.Open(ctx, source.Backend, storage.OpenConfig{storagePathKey: source.Path})
return f.registry.Open(ctx, source.Backend, sourceOpenConfig(source))
}
func (f *backendFactory) openDestination(ctx context.Context, destination config.Destination) (storage.Backend, error) {
if destination.Backend != config.BackendLocal {
if destination.Backend != config.BackendLocal && destination.Backend != config.BackendSSH {
return nil, fmt.Errorf("backend %s is not implemented for execution", destination.Backend)
}
return f.registry.Open(ctx, destination.Backend, storage.OpenConfig{storagePathKey: destination.Path})
return f.registry.Open(ctx, destination.Backend, destinationOpenConfig(destination))
}
func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storage.Backend, error) {
return f.registry.Open(ctx, config.BackendLocal, storage.OpenConfig{storagePathKey: path})
}
func sourceOpenConfig(source config.Backend) storage.OpenConfig {
cfg := storage.OpenConfig{storagePathKey: source.Path}
if source.Backend == config.BackendSSH {
cfg[sshHostKey] = source.Host
cfg[sshUserKey] = source.User
cfg[sshPortKey] = strconv.Itoa(source.Port)
cfg[sshKeyFileKey] = source.SSH.KeyFile
cfg[sshKnownHostsKey] = source.SSH.KnownHosts
cfg[sshHostKeyPolicyKey] = string(source.SSH.HostKeyPolicy)
}
return cfg
}
func destinationOpenConfig(destination config.Destination) storage.OpenConfig {
cfg := storage.OpenConfig{storagePathKey: destination.Path}
if destination.Backend == config.BackendSSH {
cfg[sshHostKey] = destination.Host
cfg[sshUserKey] = destination.User
cfg[sshPortKey] = strconv.Itoa(destination.Port)
cfg[sshKeyFileKey] = destination.SSH.KeyFile
cfg[sshKnownHostsKey] = destination.SSH.KnownHosts
cfg[sshHostKeyPolicyKey] = string(destination.SSH.HostKeyPolicy)
}
return cfg
}

View File

@@ -6,6 +6,8 @@ import (
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
)
func TestBackendFactoryOpensLocalSource(t *testing.T) {
@@ -47,14 +49,72 @@ func TestBackendFactoryOpensDirectLocalPath(t *testing.T) {
}
}
func TestBackendFactoryOpensSSHSourceWithRegisteredOpener(t *testing.T) {
factory := &backendFactory{registry: storage.NewRegistry()}
var got storage.OpenConfig
if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
got = cfg
return fake.New(), nil
}); err != nil {
t.Fatalf("Register() error = %v", err)
}
backend, err := factory.openSource(context.Background(), config.Backend{
Backend: config.BackendSSH,
Host: "source.example.com",
User: "reports",
Port: 22,
Path: "/reports",
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
})
if err != nil {
t.Fatalf("openSource() error = %v", err)
}
if backend == nil {
t.Fatal("openSource() backend = nil")
}
if got[sshHostKey] != "source.example.com" || got[storagePathKey] != "/reports" {
t.Fatalf("open config = %#v, want SSH source fields", got)
}
}
func TestBackendFactoryOpensSSHDestinationWithRegisteredOpener(t *testing.T) {
factory := &backendFactory{registry: storage.NewRegistry()}
var got storage.OpenConfig
if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
got = cfg
return fake.New(), nil
}); err != nil {
t.Fatalf("Register() error = %v", err)
}
backend, err := factory.openDestination(context.Background(), config.Destination{
Backend: config.BackendSSH,
Host: "destination.example.com",
User: "deploy",
Port: 2222,
Path: "/archive",
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyStrict},
})
if err != nil {
t.Fatalf("openDestination() error = %v", err)
}
if backend == nil {
t.Fatal("openDestination() backend = nil")
}
if got[sshHostKey] != "destination.example.com" || got[sshPortKey] != "2222" || got[sshHostKeyPolicyKey] != "strict" {
t.Fatalf("open config = %#v, want SSH destination fields", got)
}
}
func TestBackendFactoryRejectsUnsupportedSource(t *testing.T) {
factory := newBackendFactory()
_, err := factory.openSource(context.Background(), config.Backend{
Backend: config.BackendSSH,
URI: "ssh://reports@example.com:22",
Path: "/reports",
Backend: config.BackendS3,
Endpoint: "https://s3.example.com",
Bucket: "reports",
})
if err == nil || !strings.Contains(err.Error(), "source backend ssh is not implemented for execution") {
if err == nil || !strings.Contains(err.Error(), "source backend s3 is not implemented for execution") {
t.Fatalf("openSource() error = %v, want not implemented", err)
}
}
@@ -70,3 +130,58 @@ func TestBackendFactoryRejectsUnsupportedDestination(t *testing.T) {
t.Fatalf("openDestination() error = %v, want not implemented", err)
}
}
func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) {
cfg := sourceOpenConfig(config.Backend{
Backend: config.BackendSSH,
Host: "source.example.com",
User: "reports",
Port: 2222,
Path: "/reports",
SSH: config.SSH{
KeyFile: "/home/reports/.ssh/id_ed25519",
KnownHosts: "/home/reports/.ssh/known_hosts",
HostKeyPolicy: config.HostKeyPolicyStrict,
},
})
assertOpenConfig(t, cfg, map[string]string{
storagePathKey: "/reports",
sshHostKey: "source.example.com",
sshUserKey: "reports",
sshPortKey: "2222",
sshKeyFileKey: "/home/reports/.ssh/id_ed25519",
sshKnownHostsKey: "/home/reports/.ssh/known_hosts",
sshHostKeyPolicyKey: "strict",
})
}
func TestBackendFactoryBuildsSSHDestinationOpenConfig(t *testing.T) {
cfg := destinationOpenConfig(config.Destination{
Backend: config.BackendSSH,
Host: "destination.example.com",
User: "deploy",
Port: 22,
Path: "/srv/archive",
SSH: config.SSH{
HostKeyPolicy: config.HostKeyPolicyAcceptNew,
},
})
assertOpenConfig(t, cfg, map[string]string{
storagePathKey: "/srv/archive",
sshHostKey: "destination.example.com",
sshUserKey: "deploy",
sshPortKey: "22",
sshHostKeyPolicyKey: "accept-new",
})
}
func assertOpenConfig(t *testing.T, got map[string]string, want map[string]string) {
t.Helper()
for key, wantValue := range want {
if gotValue := got[key]; gotValue != wantValue {
t.Fatalf("open config %s = %q, want %q", key, gotValue, wantValue)
}
}
}

View File

@@ -52,16 +52,23 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
}
}
for _, pipeline := range cfg.Pipelines {
if options.Stdout != nil {
if err := writeSSHWarnings(options.Stdout, pipeline); err != nil {
return err
}
}
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
if err != nil {
return fmt.Errorf("pipeline %s: %w", pipeline.ID, err)
}
bundles, err := bundle.Discover(ctx, sourceBackend, "")
if err != nil {
closeBackend(sourceBackend)
return fmt.Errorf("pipeline %s discover source bundles: %w", pipeline.ID, err)
}
if options.Stdout != nil {
if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
closeBackend(sourceBackend)
return err
}
}
@@ -76,6 +83,13 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
}
continue
}
closeDestination := true
deferCloseDestination := func() {
if closeDestination {
closeBackend(destinationBackend)
closeDestination = false
}
}
req := publish.Request{
PipelineID: pipeline.ID,
DestinationID: destination.ID,
@@ -97,6 +111,7 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
writePlanLine(options.Stdout, plan, err)
}
if err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
@@ -104,20 +119,24 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
summary.recordPlan(plan.Action)
if !options.DryRun {
if err := publish.Execute(ctx, req, plan); err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
}
if shouldNotify(plan.Action) {
if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
deferCloseDestination()
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
summary.recordFailure()
continue
}
}
}
deferCloseDestination()
}
}
closeBackend(sourceBackend)
}
if options.Stdout != nil {
if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil {
@@ -130,6 +149,18 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
return nil
}
type closeableBackend interface {
Close() error
}
func closeBackend(backend storage.Backend) {
closeable, ok := backend.(closeableBackend)
if !ok {
return
}
_ = closeable.Close()
}
func writePlanLine(w io.Writer, plan publish.Plan, planErr error) {
if w == nil {
return
@@ -174,6 +205,22 @@ func destinationSummary(destinations []config.Destination) string {
return strings.Join(ids, ",")
}
func writeSSHWarnings(w io.Writer, pipeline config.Pipeline) error {
if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
if _, err := fmt.Fprintf(w, "Warning: pipeline=%s source host_key_policy=off disables SSH host key checking\n", pipeline.ID); err != nil {
return err
}
}
for _, destination := range pipeline.Destinations {
if destination.Backend == config.BackendSSH && destination.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
if _, err := fmt.Fprintf(w, "Warning: pipeline=%s destination=%s host_key_policy=off disables SSH host key checking\n", pipeline.ID, destination.ID); err != nil {
return err
}
}
}
return nil
}
func shouldNotify(action publish.Action) bool {
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder
}

View File

@@ -11,6 +11,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/notify"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
@@ -46,6 +47,34 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) {
}
}
func TestWriteSSHWarningsReportsInsecureHostKeyPolicy(t *testing.T) {
var stdout bytes.Buffer
err := writeSSHWarnings(&stdout, config.Pipeline{
ID: "reports",
Source: config.Backend{
Backend: config.BackendSSH,
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff},
},
Destinations: []config.Destination{{
ID: "archive",
Backend: config.BackendSSH,
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff},
}},
})
if err != nil {
t.Fatalf("writeSSHWarnings() error = %v", err)
}
output := stdout.String()
for _, want := range []string{
"pipeline=reports source host_key_policy=off disables SSH host key checking",
"pipeline=reports destination=archive host_key_policy=off disables SSH host key checking",
} {
if !strings.Contains(output, want) {
t.Fatalf("output = %q, want substring %q", output, want)
}
}
}
func TestRunPublishesNewLocalBundle(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()

View File

@@ -14,6 +14,9 @@ type Pipeline struct {
type Destination struct {
ID string `yaml:"id"`
Backend string `yaml:"backend"`
Host string `yaml:"host"`
User string `yaml:"user"`
Port int `yaml:"port"`
Path string `yaml:"path"`
URI string `yaml:"uri"`
Endpoint string `yaml:"endpoint"`
@@ -22,6 +25,7 @@ type Destination struct {
Region string `yaml:"region"`
ForcePath bool `yaml:"force_path_style"`
Creds Credentials `yaml:"credentials"`
SSH SSH `yaml:",inline"`
Publish *PublishPolicy `yaml:"publish"`
Transform Transform `yaml:"transform"`
Transfer TransferPolicy `yaml:"transfer"`
@@ -29,6 +33,9 @@ type Destination struct {
type Backend struct {
Backend string `yaml:"backend"`
Host string `yaml:"host"`
User string `yaml:"user"`
Port int `yaml:"port"`
Path string `yaml:"path"`
URI string `yaml:"uri"`
Endpoint string `yaml:"endpoint"`
@@ -37,6 +44,13 @@ type Backend struct {
Region string `yaml:"region"`
ForcePath bool `yaml:"force_path_style"`
Creds Credentials `yaml:"credentials"`
SSH SSH `yaml:",inline"`
}
type SSH struct {
KeyFile string `yaml:"ssh_key_file"`
KnownHosts string `yaml:"known_hosts"`
HostKeyPolicy HostKeyPolicy `yaml:"host_key_policy"`
}
type Credentials struct {

View File

@@ -25,11 +25,13 @@ const (
func ApplyDefaults(cfg *Config) {
for pipelineIndex := range cfg.Pipelines {
pipeline := &cfg.Pipelines[pipelineIndex]
applyBackendDefaults(&pipeline.Source)
if pipeline.Validation.OnDigestMismatch == "" {
pipeline.Validation.OnDigestMismatch = ValidationActionFail
}
for destinationIndex := range pipeline.Destinations {
destination := &pipeline.Destinations[destinationIndex]
applyDestinationDefaults(destination)
if destination.Publish == nil {
destination.Publish = &PublishPolicy{Source: true}
}
@@ -48,3 +50,25 @@ func ApplyDefaults(cfg *Config) {
}
}
}
func applyBackendDefaults(backend *Backend) {
if backend.Backend == BackendSSH {
if backend.Port == 0 {
backend.Port = 22
}
if backend.SSH.HostKeyPolicy == "" {
backend.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew
}
}
}
func applyDestinationDefaults(destination *Destination) {
if destination.Backend == BackendSSH {
if destination.Port == 0 {
destination.Port = 22
}
if destination.SSH.HostKeyPolicy == "" {
destination.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew
}
}
}

View File

@@ -53,7 +53,9 @@ pipelines:
html: false
- id: static-site
backend: ssh
uri: ssh://deploy@example.com:22
host: example.com
user: deploy
port: 22
path: /srv/www/reports
publish:
source: false
@@ -87,13 +89,19 @@ pipelines:
- id: ssh-backend
source:
backend: ssh
uri: ssh://reports@example.com:22
host: source.example.com
user: reports
path: /source
destinations:
- id: ssh-destination
backend: ssh
uri: ssh://deploy@example.com:22
host: destination.example.com
user: deploy
port: 2222
path: /destination
ssh_key_file: /home/deploy/.ssh/id_ed25519
known_hosts: /home/deploy/.ssh/known_hosts
host_key_policy: strict
`,
"s3": `
pipelines:
@@ -171,7 +179,7 @@ func TestLoadFileRejectsMissingRequiredFields(t *testing.T) {
"destinations": `pipelines: [{id: reports, source: {backend: local, path: /source}}]`,
"destination id": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{backend: local, path: /archive}]}]`,
"local path": `pipelines: [{id: reports, source: {backend: local}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"ssh uri": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"ssh host": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"s3 bucket": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com"}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"publish outputs": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, publish: {source: false, html: false}}]}]`,
}
@@ -183,6 +191,88 @@ func TestLoadFileRejectsMissingRequiredFields(t *testing.T) {
}
}
func TestLoadFileDefaultsSSHConfig(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: ssh-defaults
source:
backend: ssh
host: source.example.com
path: /source
destinations:
- id: archive
backend: ssh
host: destination.example.com
path: /archive
host_key_policy: false
`)
source := cfg.Pipelines[0].Source
if source.Port != 22 {
t.Fatalf("source port = %d, want 22", source.Port)
}
if source.SSH.HostKeyPolicy != HostKeyPolicyAcceptNew {
t.Fatalf("source host key policy = %q, want accept-new", source.SSH.HostKeyPolicy)
}
destination := cfg.Pipelines[0].Destinations[0]
if destination.Port != 22 {
t.Fatalf("destination port = %d, want 22", destination.Port)
}
if destination.SSH.HostKeyPolicy != HostKeyPolicyOff {
t.Fatalf("destination host key policy = %q, want off", destination.SSH.HostKeyPolicy)
}
}
func TestLoadFileNormalizesSSHHostKeyPolicies(t *testing.T) {
tests := map[string]HostKeyPolicy{
`true`: HostKeyPolicyStrict,
`"true"`: HostKeyPolicyStrict,
`strict`: HostKeyPolicyStrict,
`accept-new`: HostKeyPolicyAcceptNew,
`false`: HostKeyPolicyOff,
`"false"`: HostKeyPolicyOff,
`off`: HostKeyPolicyOff,
`"STRICT"`: HostKeyPolicyStrict,
`"ACCEPT-NEW"`: HostKeyPolicyAcceptNew,
`"OFF"`: HostKeyPolicyOff,
}
for value, want := range tests {
t.Run(value, func(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: ssh-policy
source:
backend: ssh
host: source.example.com
path: /source
host_key_policy: `+value+`
destinations:
- id: archive
backend: local
path: /archive
`)
if got := cfg.Pipelines[0].Source.SSH.HostKeyPolicy; got != want {
t.Fatalf("host key policy = %q, want %q", got, want)
}
})
}
}
func TestLoadFileRejectsSSHURIExecutionConfig(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: ssh
uri: ssh://reports@example.com:22
path: /source
destinations:
- id: archive
backend: local
path: /archive
`, "uri is not supported for ssh backend")
}
func TestLoadFileRejectsUnsupportedBackend(t *testing.T) {
assertLoadError(t, `
pipelines:
@@ -267,6 +357,7 @@ func TestExampleConfigsLoad(t *testing.T) {
"../../examples/local-publish.yml",
"../../examples/local-html.yml",
"../../examples/fan-out.yml",
"../../examples/ssh-destination.yml",
} {
t.Run(path, func(t *testing.T) {
if _, err := LoadFile(path); err != nil {

64
internal/config/ssh.go Normal file
View File

@@ -0,0 +1,64 @@
package config
import (
"fmt"
"strings"
"gopkg.in/yaml.v3"
)
type HostKeyPolicy string
const (
HostKeyPolicyStrict HostKeyPolicy = "strict"
HostKeyPolicyAcceptNew HostKeyPolicy = "accept-new"
HostKeyPolicyOff HostKeyPolicy = "off"
)
func (p *HostKeyPolicy) UnmarshalYAML(value *yaml.Node) error {
switch value.Kind {
case yaml.ScalarNode:
default:
return fmt.Errorf("host_key_policy must be a boolean or string")
}
switch value.Tag {
case "!!bool":
var enabled bool
if err := value.Decode(&enabled); err != nil {
return err
}
if enabled {
*p = HostKeyPolicyStrict
} else {
*p = HostKeyPolicyOff
}
return nil
case "!!str":
var raw string
if err := value.Decode(&raw); err != nil {
return err
}
normalized, ok := NormalizeHostKeyPolicy(raw)
if !ok {
return fmt.Errorf("host_key_policy must be strict, true, accept-new, off, or false")
}
*p = normalized
return nil
default:
return fmt.Errorf("host_key_policy must be a boolean or string")
}
}
func NormalizeHostKeyPolicy(value string) (HostKeyPolicy, bool) {
switch strings.ToLower(value) {
case "", string(HostKeyPolicyAcceptNew):
return HostKeyPolicyAcceptNew, true
case string(HostKeyPolicyStrict), "true":
return HostKeyPolicyStrict, true
case string(HostKeyPolicyOff), "false":
return HostKeyPolicyOff, true
default:
return "", false
}
}

View File

@@ -37,7 +37,7 @@ func Validate(cfg Config) error {
pipelineIDs[pipeline.ID] = struct{}{}
}
errs = validateBackend(errs, pipelineContext+".source", pipeline.Source.Backend, pipeline.Source.Path, pipeline.Source.URI, pipeline.Source.Endpoint, pipeline.Source.Bucket)
errs = validateSourceBackend(errs, pipelineContext+".source", pipeline.Source)
errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation)
if len(pipeline.Destinations) == 0 {
errs = append(errs, pipelineContext+".destinations is required")
@@ -56,7 +56,7 @@ func Validate(cfg Config) error {
destinationIDs[destination.ID] = struct{}{}
}
errs = validateBackend(errs, destinationContext, destination.Backend, destination.Path, destination.URI, destination.Endpoint, destination.Bucket)
errs = validateDestinationBackend(errs, destinationContext, destination)
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
}
@@ -68,7 +68,15 @@ func Validate(cfg Config) error {
return nil
}
func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoint, bucket string) ValidationErrors {
func validateSourceBackend(errs ValidationErrors, context string, backend Backend) ValidationErrors {
return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.URI, backend.Endpoint, backend.Bucket, backend.SSH.HostKeyPolicy)
}
func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors {
return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.URI, destination.Endpoint, destination.Bucket, destination.SSH.HostKeyPolicy)
}
func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, uri, endpoint, bucket string, hostKeyPolicy HostKeyPolicy) ValidationErrors {
switch backend {
case "":
errs = append(errs, context+".backend is required")
@@ -77,12 +85,26 @@ func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoin
errs = append(errs, context+".path is required for local backend")
}
case BackendSSH:
if uri == "" {
errs = append(errs, context+".uri is required for ssh backend")
if host == "" {
errs = append(errs, context+".host is required for ssh backend")
}
if path == "" {
errs = append(errs, context+".path is required for ssh backend")
}
if uri != "" {
errs = append(errs, context+".uri is not supported for ssh backend; use host, user, port, and path")
}
if port < 0 || port > 65535 {
errs = append(errs, context+".port must be between 1 and 65535")
}
if port == 0 {
errs = append(errs, context+".port is required for ssh backend after defaults are applied")
}
if hostKeyPolicy != "" {
if _, ok := NormalizeHostKeyPolicy(string(hostKeyPolicy)); !ok {
errs = append(errs, context+".host_key_policy must be strict, true, accept-new, off, or false")
}
}
case BackendS3:
if endpoint == "" {
errs = append(errs, context+".endpoint is required for s3 backend")