From 84f77ec0d08120f3868c4498f276a708c45fdac1 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 31 May 2026 16:53:37 +0000 Subject: [PATCH] Add SSH SFTP backend support --- README.md | 2 +- docs/cli.md | 4 +- docs/config.md | 39 +- docs/internal/app.md | 4 +- docs/internal/config.md | 6 +- docs/internal/storage.md | 7 +- docs/operations.md | 26 +- docs/policy/development.md | 8 +- docs/troubleshooting.md | 70 +++- examples/ssh-destination.yml | 21 + go.mod | 7 + go.sum | 16 + internal/adapters/ssh/auth.go | 61 +++ internal/adapters/ssh/auth_test.go | 59 +++ internal/adapters/ssh/backend.go | 445 ++++++++++++++++++++++ internal/adapters/ssh/hostkeys.go | 81 ++++ internal/adapters/ssh/hostkeys_test.go | 87 +++++ internal/adapters/ssh/integration_test.go | 39 ++ internal/adapters/ssh/options.go | 77 ++++ internal/adapters/ssh/options_test.go | 118 ++++++ internal/app/backends.go | 60 ++- internal/app/backends_test.go | 123 +++++- internal/app/run.go | 47 +++ internal/app/run_test.go | 29 ++ internal/config/config.go | 14 + internal/config/defaults.go | 24 ++ internal/config/load_test.go | 99 ++++- internal/config/ssh.go | 64 ++++ internal/config/validate.go | 32 +- 29 files changed, 1629 insertions(+), 40 deletions(-) create mode 100644 examples/ssh-destination.yml create mode 100644 internal/adapters/ssh/auth.go create mode 100644 internal/adapters/ssh/auth_test.go create mode 100644 internal/adapters/ssh/backend.go create mode 100644 internal/adapters/ssh/hostkeys.go create mode 100644 internal/adapters/ssh/hostkeys_test.go create mode 100644 internal/adapters/ssh/integration_test.go create mode 100644 internal/adapters/ssh/options.go create mode 100644 internal/adapters/ssh/options_test.go create mode 100644 internal/config/ssh.go diff --git a/README.md b/README.md index 10c4dd9..060d954 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ `distributor` validates manifested report bundles and publishes selected source or generated artifacts to configured destinations. -It is currently a local-first CLI: source bundles are read from local storage, destinations are local directories, and Markdown files can be rendered to HTML sidecars. +It is a local-first CLI with SSH/SFTP support: source bundles can be read from local or SSH storage, destinations can be local directories or SSH paths, and Markdown files can be rendered to HTML sidecars. Run the local example pipeline: diff --git a/docs/cli.md b/docs/cli.md index e954482..03be4c8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -19,11 +19,11 @@ distributor inspect ``` - `version`: prints the application name and version. Development builds print `distributor dev`. -- `run`: loads a YAML config, discovers local source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary. +- `run`: loads a YAML config, discovers source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary. - `validate`: validates a local source bundle directory or a local tree containing source bundles. - `inspect`: validates local source bundles and prints normalized bundle metadata. -`validate` and `inspect` accept local paths only. `run` currently executes local backends only. SSH and S3 config can be parsed and validated, but configured SSH or S3 execution fails with a clear unsupported-execution error. +`validate` and `inspect` accept local paths only. `run` executes `local` and `ssh` backends. S3 config can be parsed and validated, but configured S3 execution fails with a clear unsupported-execution error. ## Flag reference diff --git a/docs/config.md b/docs/config.md index da36cd0..5e0c037 100644 --- a/docs/config.md +++ b/docs/config.md @@ -10,7 +10,7 @@ If `--config` is omitted, `run` uses: /usr/local/etc/distributor/config.yml ``` -Config parsing rejects unknown YAML fields. The current executable backend support is local only. SSH and S3 config fields are accepted by config validation, but runtime execution for those backends is unavailable. +Config parsing rejects unknown YAML fields. The executable backends are `local` and `ssh`. S3 config fields are accepted by config validation, but runtime execution for S3 is unavailable. ## Minimal Local Config @@ -85,7 +85,12 @@ Source backend: - `backend`: required. - `path`: required for `local` and `ssh`. -- `uri`: required for `ssh`. +- `host`: required for `ssh`. +- `user`: optional for `ssh`; defaults to the current OS user when available. +- `port`: optional for `ssh`; defaults to `22`. +- `ssh_key_file`: optional for `ssh`. +- `known_hosts`: optional for `ssh`; defaults to the service user's OpenSSH `known_hosts` path when available. +- `host_key_policy`: optional for `ssh`; defaults to `accept-new`. - `endpoint`: required for `s3`. - `bucket`: required for `s3`. - `prefix`: optional for `s3`. @@ -105,9 +110,34 @@ Destination: Accepted backend names: - `local`: executable; requires `path`. -- `ssh`: config validation only; execution is unavailable. +- `ssh`: executable; requires `host` and `path`. - `s3`: config validation only; execution is unavailable. +## SSH Backend + +SSH uses native SFTP. It can be used for sources, destinations, or both: + +```yaml +backend: ssh +host: example.com +user: distributor +port: 2222 +path: /remote/root +ssh_key_file: /home/distributor/.ssh/id_ed25519 +known_hosts: /home/distributor/.ssh/known_hosts +host_key_policy: accept-new +``` + +Authentication uses SSH agent identities first when `SSH_AUTH_SOCK` is set, then `ssh_key_file` if configured. Password authentication in YAML is not supported. + +Host key policies: + +- `strict`, `true`, and `"true"` require a matching known host key. +- `accept-new` accepts and persists a new host key, but fails if an existing key changed. +- `off`, `false`, and `"false"` disable host key checking and are insecure. + +`accept-new` and `strict` use `known_hosts` when configured. If omitted, distributor uses the current service user's default OpenSSH `known_hosts` path where practical. `accept-new` fails when it needs to persist a new host key and no writable `known_hosts` path is available. It does not create a missing parent `.ssh` directory. + Publish policy: - `publish.source`: publish source artifacts. @@ -127,6 +157,8 @@ Transfer policy: Defaults are applied after YAML decoding and before validation: - `validation.on_digest_mismatch: fail` +- SSH `port: 22` +- SSH `host_key_policy: accept-new` - `publish.source: true` - `publish.html: false` - `transfer.on_destination_same: skip` @@ -151,3 +183,4 @@ Maintained examples live under [examples](../examples/): - `local-publish.yml`: runnable local source publication. - `local-html.yml`: runnable local HTML publication. - `fan-out.yml`: runnable local fan-out publication to source and HTML destinations. +- `ssh-destination.yml`: environment-gated local-to-SSH publication example. diff --git a/docs/internal/app.md b/docs/internal/app.md index 10afb79..aeb4ffa 100644 --- a/docs/internal/app.md +++ b/docs/internal/app.md @@ -27,7 +27,7 @@ Destination failures are collected while later destinations continue to run. Sou ## Backend and transform wiring -The app-level backend factory registers only the local backend for execution. Config validation accepts other backend shapes, but `Run` can execute only local sources and local destinations. +The app-level backend factory registers local and SSH backends for execution. Config validation accepts S3 shape, but `Run` cannot execute S3 sources or destinations. The app-level transform registry registers Markdown-to-HTML using `internal/transform/markdown`. Lower-level publish code receives a resolver and does not import concrete transform implementations. @@ -45,7 +45,7 @@ Stdout write errors are returned immediately because the caller's requested outp `internal/app` coordinates packages but does not own manifest validation rules, destination state comparison, storage path rules, output planning, transform rendering, or backend-specific filesystem behavior. -`Validate` and `Inspect` are local path commands. Remote execution wiring is outside current behavior. +`Validate` and `Inspect` are local path commands. Remote execution wiring currently belongs to `Run`. ## Tests diff --git a/docs/internal/config.md b/docs/internal/config.md index 6f31dc1..fe1d009 100644 --- a/docs/internal/config.md +++ b/docs/internal/config.md @@ -19,6 +19,8 @@ Known-field checking rejects misspelled or unknown YAML keys before defaults and Defaults are applied in `ApplyDefaults`: - pipeline validation defaults `on_digest_mismatch` to `fail`; +- SSH backend `port` defaults to `22`; +- SSH backend `host_key_policy` defaults to `accept-new`; - destination publish policy defaults to source output only; - `transfer.on_destination_same` defaults to `skip`; - `transfer.on_destination_older` defaults to `replace`; @@ -33,7 +35,9 @@ Validation requires at least one pipeline, slug-like unique pipeline ids, one so ## Executable support boundary -Config validation accepts `local`, `ssh`, and `s3` backend shapes so config files can be validated as schemas. Runtime execution currently opens only local backends through `internal/app`. +Config validation accepts `local`, `ssh`, and `s3` backend shapes so config files can be validated as schemas. Runtime execution opens local and SSH backends through `internal/app`; S3 remains accepted by validation but unavailable at execution. + +SSH config uses structured fields: `host`, optional `user`, optional `port`, `path`, optional `ssh_key_file`, optional `known_hosts`, and optional `host_key_policy`. `host_key_policy` accepts YAML booleans and strings and normalizes `true`/`strict`, `accept-new`, and `false`/`off`. The user-facing configuration reference is `docs/config.md`; this file documents package behavior for maintainers. diff --git a/docs/internal/storage.md b/docs/internal/storage.md index 8e330f3..afb71a1 100644 --- a/docs/internal/storage.md +++ b/docs/internal/storage.md @@ -14,7 +14,7 @@ Entries report a logical path, type, and size when available. Entry types are `f Core packages should depend on `internal/storage`, not adapter packages. Adapter-specific path handling stays behind backend implementations. -The local adapter lives in `internal/adapters/local`. Runtime backend construction is wired through the app-level backend factory and storage registry. The fake backend lives in `internal/storage/fake` for tests and is not registered for runtime use. +The local adapter lives in `internal/adapters/local`. The SSH/SFTP adapter lives in `internal/adapters/ssh`. Runtime backend construction is wired through the app-level backend factory and storage registry. The fake backend lives in `internal/storage/fake` for tests and is not registered for runtime use. ## Paths @@ -30,10 +30,12 @@ Backends may wrap implementation-specific errors, but callers should receive sto Backends expose guarded managed deletion only. `DeleteManagedBundle` may delete listed managed outputs plus `.distributor.json`; it does not provide broad recursive deletion. -## Local and fake backends +## Local, SSH, and fake backends The local adapter maps logical paths to a configured filesystem root and keeps adapter-specific path handling behind the storage interface. +The SSH adapter maps logical paths to a configured remote SFTP root. It uses native SSH and SFTP libraries, supports SSH agent and key-file authentication, applies host-key policies, rejects unsafe logical paths, reports symlink entries from `Lstat`, and limits deletion to managed targets. + The fake backend is an in-memory implementation for package tests. It is not registered for runtime use. ## Tests @@ -43,6 +45,7 @@ Before changing storage behavior, inspect tests under: - `internal/storage` - `internal/storage/fake` - `internal/adapters/local` +- `internal/adapters/ssh` ## Invariants diff --git a/docs/operations.md b/docs/operations.md index 189d032..59321f7 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -32,13 +32,21 @@ Preview local fan-out publication: go run ./cmd/distributor run --config examples/fan-out.yml --dry-run ``` +Preview an environment-gated SSH destination config after editing it for an SSH/SFTP endpoint you control: + +```sh +go run ./cmd/distributor run --config examples/ssh-destination.yml --dry-run +``` + ## Filesystem Layout -Source bundles are discovered beneath the configured local source root. Each bundle is a directory containing `manifest.json`. +Source bundles are discovered beneath the configured source root. Each bundle is a directory containing `manifest.json`. Destination bundle paths preserve the source bundle path relative to the source root. A source bundle at the source root publishes to the destination root. A source bundle under `daily/` publishes under `daily/` at each destination. -The maintained examples write under `workspace/`, which is ignored by Git. +The maintained local examples write under `workspace/`, which is ignored by Git. + +SSH backends use the configured remote `path` as the backend root. Source bundle discovery and destination bundle paths are relative to that root, using the same logical path rules as local storage. ## Destination State @@ -74,12 +82,22 @@ If a destination path has files but no valid `.distributor.json`, publication fa If one destination fails in a fan-out run, independent later destinations are still planned and executed. The command exits non-zero after printing the final status if any destination failed. -If a write fails during local publication, `distributor` attempts to remove outputs written during that failed attempt so a retry does not see those partial outputs as unmanaged destination content. +If a write fails during publication, `distributor` attempts to remove outputs written during that failed attempt so a retry does not see those partial outputs as unmanaged destination content. After a successful publish or replacement, the internal notifier hook runs. The current default notifier is a no-op. Skipped destinations do not invoke it. +## SSH Operation Notes + +SSH execution uses SFTP over `golang.org/x/crypto/ssh` and `github.com/pkg/sftp`. It does not shell out to `ssh`, `scp`, or `rsync`. + +Configure `ssh_key_file`, an SSH agent, or both. Agent identities are attempted first, followed by the configured key file. YAML password authentication is not supported. + +The default host key policy is `accept-new`. New host keys are written to `known_hosts` when the file path is writable. Changed host keys are fatal for both `strict` and `accept-new`. The `off` policy disables host key checking and `run` prints a warning when stdout is enabled. + +Recovery boundaries are the same as local storage: replacement deletes only managed output paths recorded in `.distributor.json` plus the state file, and failed writes are cleaned up where practical. Distributor never performs broad recursive remote deletion. + ## Caveats -Only local-to-local execution is available. SSH execution, S3 execution, external notification adapters, and force overwrite behavior are unavailable. +S3 execution, external notification adapters, and force overwrite behavior are unavailable. For symptom-oriented fixes, see [troubleshooting](troubleshooting.md). For config details, see [configuration](config.md). For command syntax, see [CLI](cli.md). diff --git a/docs/policy/development.md b/docs/policy/development.md index 7f323bd..3402b83 100644 --- a/docs/policy/development.md +++ b/docs/policy/development.md @@ -102,7 +102,7 @@ When adding or changing configuration: Config validation may accept fields for backends that are not executable yet, but user-facing docs and examples must clearly state execution support. At the -time of this policy, only the local backend is executable. +time of this policy, local and SSH backends are executable. ## CLI Changes @@ -117,7 +117,7 @@ When adding or changing commands or flags: 4. Update `docs/cli.md` if syntax, flags, output expectations, or workflows change. `validate` and `inspect` are local path commands. `run` loads configured -pipelines and currently executes local backends only. +pipelines and currently executes local and SSH backends. ## Storage Backends @@ -133,8 +133,8 @@ When adding a backend: 5. Add focused adapter tests and app-level wiring tests. 6. Update user docs, operations docs, examples, and internal docs only for behavior that is actually implemented. -Do not document SSH/SFTP or S3 execution as available until corresponding -adapter packages and app wiring exist. +Do not document S3 execution as available until the corresponding adapter +package and app wiring exist. ## Transforms diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a14a453..654b8ed 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -34,11 +34,11 @@ Diagnostic: rg -n "backend:" ``` -Safe fix: use `backend: local` for executable workflows. SSH and S3 config shapes are accepted only for validation; runtime execution is unavailable. +Safe fix: use `backend: local` or `backend: ssh` for executable workflows. S3 config shape is accepted only for validation; runtime execution is unavailable. -## `backend ssh is not implemented for execution` or `backend s3 is not implemented for execution` +## `backend s3 is not implemented for execution` -Likely cause: the config validates but `run` tried to execute a remote backend. +Likely cause: the config validates but `run` tried to execute S3, which is not implemented. Diagnostic: @@ -46,7 +46,69 @@ Diagnostic: go run ./cmd/distributor run --config --dry-run ``` -Safe fix: use local destinations for current executable workflows, or keep remote backend configs under roadmap material unless those adapters are added. See [configuration](config.md). +Safe fix: use `local` or `ssh` for executable workflows. See [configuration](config.md). + +## `host is required for ssh backend` + +Likely cause: SSH config is missing the structured `host` field, or an old URI-based SSH config is still in use. + +Diagnostic: + +```sh +go run ./cmd/distributor run --config --dry-run +``` + +Safe fix: configure SSH with `host`, optional `user` and `port`, and `path`. The `uri` field is not used for SSH execution. + +## `no SSH auth methods configured` + +Likely cause: neither an SSH agent nor `ssh_key_file` is available. + +Diagnostic: + +```sh +test -n "$SSH_AUTH_SOCK" && ssh-add -l +ls -l +``` + +Safe fix: start an SSH agent with an appropriate key loaded, or configure `ssh_key_file` with a readable private key. + +## `host key ... is unknown` or `known_hosts is required` + +Likely cause: strict host key checking has no known host key, or `accept-new` cannot persist a new key. + +Diagnostic: + +```sh +ls -l +ssh-keygen -F -f +``` + +Safe fix: configure a writable `known_hosts` path for `accept-new`, pre-populate `known_hosts` for `strict`, or explicitly use `host_key_policy: off` only for insecure test environments. + +## `host key ... has changed` + +Likely cause: the remote server presented a different host key than the one recorded in `known_hosts`. + +Diagnostic: + +```sh +ssh-keygen -F -f +``` + +Safe fix: verify the server identity out of band before updating `known_hosts`. Do not switch to `host_key_policy: off` to bypass an unexpected changed key. + +## `stat ssh ... not_found` or `no bundles found` + +Likely cause: the configured SSH `path` is wrong, unreadable, or does not contain source bundles. + +Diagnostic: + +```sh +sftp @ +``` + +Safe fix: correct the remote root `path`, permissions, or source bundle location. ## `validate command requires a path` or `inspect command requires a path` diff --git a/examples/ssh-destination.yml b/examples/ssh-destination.yml new file mode 100644 index 0000000..6c907b7 --- /dev/null +++ b/examples/ssh-destination.yml @@ -0,0 +1,21 @@ +# Environment-gated example. +# Replace host, user, path, ssh_key_file, and known_hosts with values for an +# SSH/SFTP endpoint you control before running this config. +pipelines: + - id: example-ssh-destination + source: + backend: local + path: examples/source-bundle + destinations: + - id: ssh-archive + backend: ssh + host: ssh.example.com + user: distributor + port: 22 + path: /srv/distributor/archive + ssh_key_file: /home/distributor/.ssh/id_ed25519 + known_hosts: /home/distributor/.ssh/known_hosts + host_key_policy: strict + publish: + source: true + html: false diff --git a/go.mod b/go.mod index b84ddb7..b44c1e5 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,13 @@ module gitea.maximumdirect.net/eric/distributor go 1.26 require ( + github.com/pkg/sftp v1.13.10 github.com/yuin/goldmark v1.8.2 + golang.org/x/crypto v0.52.0 gopkg.in/yaml.v3 v3.0.1 ) + +require ( + github.com/kr/fs v0.1.0 // indirect + golang.org/x/sys v0.45.0 // indirect +) diff --git a/go.sum b/go.sum index f3e50ca..a513948 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,21 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= +github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/adapters/ssh/auth.go b/internal/adapters/ssh/auth.go new file mode 100644 index 0000000..f72ccf1 --- /dev/null +++ b/internal/adapters/ssh/auth.go @@ -0,0 +1,61 @@ +package ssh + +import ( + "fmt" + "io" + "net" + "os" + + cryptossh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" +) + +type agentDialer func(network, address string) (net.Conn, error) + +func authMethods(keyFile string) ([]cryptossh.AuthMethod, func(), error) { + return authMethodsWithAgent(os.Getenv("SSH_AUTH_SOCK"), net.Dial, keyFile) +} + +func authMethodsWithAgent(agentSocket string, dial agentDialer, keyFile string) ([]cryptossh.AuthMethod, func(), error) { + var methods []cryptossh.AuthMethod + var closers []io.Closer + if agentSocket != "" { + methods = append(methods, cryptossh.PublicKeysCallback(func() ([]cryptossh.Signer, error) { + conn, err := dial("unix", agentSocket) + if err != nil { + return nil, err + } + closers = append(closers, conn) + return agent.NewClient(conn).Signers() + })) + } + if keyFile != "" { + signer, err := signerFromKeyFile(keyFile) + if err != nil { + return nil, nil, err + } + methods = append(methods, cryptossh.PublicKeys(signer)) + } + if len(methods) == 0 { + return nil, nil, fmt.Errorf("no SSH auth methods configured; set SSH_AUTH_SOCK or ssh_key_file") + } + return methods, func() { closeAll(closers) }, nil +} + +func signerFromKeyFile(path string) (cryptossh.Signer, error) { + key, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read ssh_key_file %q: %w", path, err) + } + signer, err := cryptossh.ParsePrivateKey(key) + if err != nil { + return nil, fmt.Errorf("parse ssh_key_file %q: %w", path, err) + } + return signer, nil +} + +func closeAll(closers []io.Closer) { + for _, closer := range closers { + _ = closer.Close() + } +} diff --git a/internal/adapters/ssh/auth_test.go b/internal/adapters/ssh/auth_test.go new file mode 100644 index 0000000..107e12c --- /dev/null +++ b/internal/adapters/ssh/auth_test.go @@ -0,0 +1,59 @@ +package ssh + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "os" + "path/filepath" + "testing" +) + +func TestAuthMethodsPreferAgentBeforeKeyFile(t *testing.T) { + keyFile := writePrivateKey(t) + methods, cleanup, err := authMethodsWithAgent("/tmp/ssh-agent.sock", nil, keyFile) + if err != nil { + t.Fatalf("authMethodsWithAgent() error = %v", err) + } + defer cleanup() + if got, want := len(methods), 2; got != want { + t.Fatalf("auth method count = %d, want %d", got, want) + } +} + +func TestAuthMethodsLoadsKeyFile(t *testing.T) { + keyFile := writePrivateKey(t) + methods, cleanup, err := authMethodsWithAgent("", nil, keyFile) + if err != nil { + t.Fatalf("authMethodsWithAgent() error = %v", err) + } + defer cleanup() + if got, want := len(methods), 1; got != want { + t.Fatalf("auth method count = %d, want %d", got, want) + } +} + +func TestAuthMethodsRejectsMissingAuth(t *testing.T) { + _, _, err := authMethodsWithAgent("", nil, "") + if err == nil { + t.Fatal("authMethodsWithAgent() error = nil, want error") + } +} + +func writePrivateKey(t *testing.T) string { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate key: %v", err) + } + data := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + }) + path := filepath.Join(t.TempDir(), "id_rsa") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write private key: %v", err) + } + return path +} diff --git a/internal/adapters/ssh/backend.go b/internal/adapters/ssh/backend.go new file mode 100644 index 0000000..345ba4d --- /dev/null +++ b/internal/adapters/ssh/backend.go @@ -0,0 +1,445 @@ +package ssh + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "sort" + "strings" + "time" + + "gitea.maximumdirect.net/eric/distributor/internal/storage" + "github.com/pkg/sftp" + cryptossh "golang.org/x/crypto/ssh" +) + +type Backend struct { + client *sftp.Client + sshClient *cryptossh.Client + root string +} + +func New(ctx context.Context, options Options) (*Backend, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + options, err := options.normalized() + if err != nil { + return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Root, storage.ErrInvalidPath, err) + } + hostKeyCallback, err := hostKeyCallback(options) + if err != nil { + return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.KnownHosts, storage.ErrInvalidPath, err) + } + auth, cleanupAuth, err := authMethods(options.KeyFile) + if err != nil { + return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.KeyFile, storage.ErrPermission, err) + } + defer cleanupAuth() + + sshClient, err := cryptossh.Dial("tcp", options.address(), &cryptossh.ClientConfig{ + User: options.User, + Auth: auth, + HostKeyCallback: hostKeyCallback, + Timeout: 30 * time.Second, + }) + if err != nil { + return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.address(), storage.ErrUnknown, err) + } + client, err := sftp.NewClient(sshClient) + if err != nil { + _ = sshClient.Close() + return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.address(), storage.ErrUnknown, err) + } + return &Backend{client: client, sshClient: sshClient, root: options.Root}, nil +} + +func (b *Backend) Close() error { + var err error + if b.client != nil { + err = b.client.Close() + } + if b.sshClient != nil { + if closeErr := b.sshClient.Close(); err == nil { + err = closeErr + } + } + return err +} + +func (b *Backend) ReadFile(ctx context.Context, logicalPath string) ([]byte, error) { + reader, err := b.OpenReader(ctx, logicalPath) + if err != nil { + return nil, err + } + defer reader.Close() + data, err := io.ReadAll(reader) + if err != nil { + return nil, storage.NewError(storage.OpReadFile, BackendName, logicalPath, storage.ErrUnknown, err) + } + return data, nil +} + +func (b *Backend) OpenReader(ctx context.Context, logicalPath string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + nativePath, err := b.nativePath(logicalPath, false) + if err != nil { + return nil, err + } + if err := b.rejectSymlinkAncestors(ctx, logicalPath, true); err != nil { + return nil, err + } + info, err := b.client.Lstat(nativePath) + if err != nil { + return nil, b.translateError(storage.OpOpenReader, logicalPath, err) + } + if !info.Mode().IsRegular() { + return nil, storage.NewError(storage.OpOpenReader, BackendName, logicalPath, storage.ErrUnsupported, nil) + } + file, err := b.client.Open(nativePath) + if err != nil { + return nil, b.translateError(storage.OpOpenReader, logicalPath, err) + } + return file, nil +} + +func (b *Backend) WriteFile(ctx context.Context, logicalPath string, data []byte, opts storage.WriteOptions) (storage.Entry, error) { + opts.Size = int64(len(data)) + opts.SizeKnown = true + return b.WriteFrom(ctx, logicalPath, bytes.NewReader(data), opts) +} + +func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) { + if err := ctx.Err(); err != nil { + return storage.Entry{}, err + } + nativePath, err := b.nativePath(logicalPath, false) + if err != nil { + return storage.Entry{}, err + } + if err := b.rejectSymlinkAncestors(ctx, parentOf(logicalPath), true); err != nil { + return storage.Entry{}, err + } + if info, err := b.client.Lstat(nativePath); err == nil { + if !opts.Overwrite { + return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrAlreadyExist, nil) + } + if !info.Mode().IsRegular() { + return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, nil) + } + } else if !isNotExist(err) { + return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err) + } + + parentNative := path.Dir(nativePath) + if err := b.client.MkdirAll(parentNative); err != nil { + return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err) + } + + writePath := nativePath + if opts.PreferAtomic { + writePath = path.Join(parentNative, fmt.Sprintf(".distributor-write-%d", time.Now().UnixNano())) + } + file, err := b.client.Create(writePath) + if err != nil { + return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err) + } + cleanup := opts.PreferAtomic + defer func() { + if cleanup { + _ = b.client.Remove(writePath) + } + }() + + written, copyErr := io.Copy(file, r) + closeErr := file.Close() + if copyErr != nil { + return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, copyErr) + } + if closeErr != nil { + return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, closeErr) + } + if opts.SizeKnown && written != opts.Size { + return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", written, opts.Size)) + } + if opts.PreferAtomic { + if err := b.client.Rename(writePath, nativePath); err != nil { + return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err) + } + cleanup = false + } + return b.Stat(ctx, logicalPath) +} + +func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) { + if err := ctx.Err(); err != nil { + return storage.Entry{}, err + } + nativePath, err := b.nativePath(logicalPath, true) + if err != nil { + return storage.Entry{}, err + } + info, err := b.client.Lstat(nativePath) + if err != nil { + return storage.Entry{}, b.translateError(storage.OpStat, logicalPath, err) + } + return entryFromInfo(logicalPath, info), nil +} + +func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOptions, fn storage.WalkFunc) error { + if err := ctx.Err(); err != nil { + return err + } + nativePrefix, err := b.nativePath(prefix, true) + if err != nil { + return err + } + info, err := b.client.Lstat(nativePrefix) + if err != nil { + if isNotExist(err) { + return nil + } + return b.translateError(storage.OpWalk, prefix, err) + } + + visited := 0 + emit := func(entry storage.Entry) error { + if err := ctx.Err(); err != nil { + return err + } + if opts.Limit > 0 && visited >= opts.Limit { + return storage.ErrStopWalk + } + visited++ + if err := fn(entry); err != nil { + if errors.Is(err, storage.ErrStopWalk) { + return storage.ErrStopWalk + } + return storage.NewError(storage.OpWalk, BackendName, entry.Path, storage.ErrUnknown, err) + } + return nil + } + + if !info.IsDir() { + if err := emit(entryFromInfo(prefix, info)); errors.Is(err, storage.ErrStopWalk) { + return nil + } else if err != nil { + return err + } + return nil + } + + if err := b.walkDirectory(ctx, prefix, nativePrefix, opts, emit); errors.Is(err, storage.ErrStopWalk) { + return nil + } else if err != nil { + return err + } + return nil +} + +func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) { + found := false + err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error { + found = true + return storage.ErrStopWalk + }) + if err != nil { + return false, err + } + return found, nil +} + +func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error { + if err := ctx.Err(); err != nil { + return err + } + targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths) + if err != nil { + return err + } + for _, target := range targets { + nativePath, err := b.nativePath(target, false) + if err != nil { + return err + } + if nativePath == b.root { + return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrInvalidPath, nil) + } + info, err := b.client.Lstat(nativePath) + if err != nil { + if opts.IgnoreMissing && isNotExist(err) { + continue + } + return b.translateError(storage.OpDeleteManagedBundle, target, err) + } + if info.IsDir() { + return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrUnsupported, nil) + } + if err := b.client.Remove(nativePath); err != nil { + return b.translateError(storage.OpDeleteManagedBundle, target, err) + } + if opts.PruneEmptyDirs { + b.pruneEmptyParents(parentOf(target)) + } + } + return nil +} + +func (b *Backend) walkDirectory(ctx context.Context, logicalPrefix, nativePrefix string, opts storage.WalkOptions, emit func(storage.Entry) error) error { + entries, err := b.client.ReadDir(nativePrefix) + if err != nil { + return b.translateError(storage.OpWalk, logicalPrefix, err) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + for _, info := range entries { + if err := ctx.Err(); err != nil { + return err + } + logicalPath := info.Name() + if logicalPrefix != "" { + logicalPath = logicalPrefix + "/" + info.Name() + } + if err := emit(entryFromInfo(logicalPath, info)); err != nil { + return err + } + if opts.Recursive && info.IsDir() { + if err := b.walkDirectory(ctx, logicalPath, path.Join(nativePrefix, info.Name()), opts, emit); err != nil { + return err + } + } + } + return nil +} + +func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) { + if logicalPath == "" { + if !allowEmpty { + return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil) + } + return b.root, nil + } + if err := storage.ValidatePath(logicalPath); err != nil { + return "", err + } + nativePath := path.Clean(path.Join(b.root, logicalPath)) + if !withinRoot(b.root, nativePath) { + return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil) + } + return nativePath, nil +} + +func (b *Backend) rejectSymlinkAncestors(ctx context.Context, logicalPath string, includeFinal bool) error { + if logicalPath == "" { + return nil + } + if err := storage.ValidatePath(logicalPath); err != nil { + return err + } + segments := strings.Split(logicalPath, "/") + limit := len(segments) + if !includeFinal { + limit-- + } + current := "" + for index := 0; index < limit; index++ { + if err := ctx.Err(); err != nil { + return err + } + if current == "" { + current = segments[index] + } else { + current += "/" + segments[index] + } + nativePath, err := b.nativePath(current, false) + if err != nil { + return err + } + info, err := b.client.Lstat(nativePath) + if err != nil { + if isNotExist(err) { + return nil + } + return b.translateError(storage.OpStat, current, err) + } + if info.Mode()&os.ModeSymlink != 0 { + return storage.NewError(storage.OpStat, BackendName, current, storage.ErrUnsupported, nil) + } + } + return nil +} + +func (b *Backend) pruneEmptyParents(logicalPath string) { + for logicalPath != "" { + nativePath, err := b.nativePath(logicalPath, false) + if err != nil || nativePath == b.root { + return + } + if err := b.client.RemoveDirectory(nativePath); err != nil { + return + } + logicalPath = parentOf(logicalPath) + } +} + +func withinRoot(root, candidate string) bool { + if candidate == root { + return true + } + if root == "/" { + return strings.HasPrefix(candidate, "/") + } + return strings.HasPrefix(candidate, strings.TrimSuffix(root, "/")+"/") +} + +func parentOf(logicalPath string) string { + index := strings.LastIndex(logicalPath, "/") + if index == -1 { + return "" + } + return logicalPath[:index] +} + +func isNotExist(err error) bool { + return errors.Is(err, fs.ErrNotExist) || errors.Is(err, os.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile) +} + +func (b *Backend) translateError(op, logicalPath string, err error) error { + kind := storage.ErrUnknown + switch { + case isNotExist(err): + kind = storage.ErrNotFound + case errors.Is(err, fs.ErrExist), errors.Is(err, os.ErrExist): + kind = storage.ErrAlreadyExist + case errors.Is(err, fs.ErrPermission), errors.Is(err, os.ErrPermission), errors.Is(err, sftp.ErrSSHFxPermissionDenied): + kind = storage.ErrPermission + case errors.Is(err, sftp.ErrSSHFxOpUnsupported): + kind = storage.ErrUnsupported + case errors.Is(err, sftp.ErrSSHFxNoConnection), errors.Is(err, sftp.ErrSSHFxConnectionLost): + kind = storage.ErrTemporary + } + return storage.NewError(op, BackendName, logicalPath, kind, err) +} + +func entryFromInfo(logicalPath string, info fs.FileInfo) storage.Entry { + entryType := storage.EntryTypeOther + switch { + case info.Mode()&os.ModeSymlink != 0: + entryType = storage.EntryTypeSymlink + case info.Mode().IsRegular(): + entryType = storage.EntryTypeFile + case info.IsDir(): + entryType = storage.EntryTypeDirectory + } + return storage.Entry{ + Path: logicalPath, + Type: entryType, + Size: info.Size(), + } +} diff --git a/internal/adapters/ssh/hostkeys.go b/internal/adapters/ssh/hostkeys.go new file mode 100644 index 0000000..828a5a7 --- /dev/null +++ b/internal/adapters/ssh/hostkeys.go @@ -0,0 +1,81 @@ +package ssh + +import ( + "errors" + "fmt" + "net" + "os" + + cryptossh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +func hostKeyCallback(options Options) (cryptossh.HostKeyCallback, error) { + switch options.HostKeyPolicy { + case HostKeyPolicyOff: + return cryptossh.InsecureIgnoreHostKey(), nil + case HostKeyPolicyStrict: + if options.KnownHosts == "" { + return nil, fmt.Errorf("known_hosts is required for strict host key checking") + } + callback, err := knownhosts.New(options.KnownHosts) + if err != nil { + return nil, fmt.Errorf("load known_hosts %q: %w", options.KnownHosts, err) + } + return callback, nil + case HostKeyPolicyAcceptNew: + return acceptNewHostKeyCallback(options) + default: + return nil, fmt.Errorf("host_key_policy must be strict, accept-new, or off") + } +} + +func acceptNewHostKeyCallback(options Options) (cryptossh.HostKeyCallback, error) { + var checker cryptossh.HostKeyCallback + if options.KnownHosts != "" { + loaded, err := knownhosts.New(options.KnownHosts) + if err == nil { + checker = loaded + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("load known_hosts %q: %w", options.KnownHosts, err) + } + } + return func(hostname string, remote net.Addr, key cryptossh.PublicKey) error { + if checker != nil { + err := checker(hostname, remote, key) + if err == nil { + return nil + } + var keyErr *knownhosts.KeyError + if !errors.As(err, &keyErr) { + return err + } + if len(keyErr.Want) > 0 { + return fmt.Errorf("host key for %s has changed: %w", hostname, err) + } + } + if options.KnownHosts == "" { + return fmt.Errorf("host key for %s is unknown and no writable known_hosts path is available", hostname) + } + if err := appendKnownHost(options.KnownHosts, hostname, key); err != nil { + return err + } + loaded, err := knownhosts.New(options.KnownHosts) + if err == nil { + checker = loaded + } + return nil + }, nil +} + +func appendKnownHost(path, host string, key cryptossh.PublicKey) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return fmt.Errorf("persist accepted host key to known_hosts %q: %w", path, err) + } + defer file.Close() + if _, err := fmt.Fprintln(file, knownhosts.Line([]string{knownhosts.Normalize(host)}, key)); err != nil { + return fmt.Errorf("persist accepted host key to known_hosts %q: %w", path, err) + } + return nil +} diff --git a/internal/adapters/ssh/hostkeys_test.go b/internal/adapters/ssh/hostkeys_test.go new file mode 100644 index 0000000..7ad756c --- /dev/null +++ b/internal/adapters/ssh/hostkeys_test.go @@ -0,0 +1,87 @@ +package ssh + +import ( + "crypto/rand" + "crypto/rsa" + "net" + "os" + "path/filepath" + "strings" + "testing" + + cryptossh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +func TestAcceptNewHostKeyCallbackPersistsUnknownHost(t *testing.T) { + key := testPublicKey(t) + knownHosts := filepath.Join(t.TempDir(), "known_hosts") + callback, err := acceptNewHostKeyCallback(Options{KnownHosts: knownHosts}) + if err != nil { + t.Fatalf("acceptNewHostKeyCallback() error = %v", err) + } + + if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil { + t.Fatalf("callback() error = %v", err) + } + data, err := os.ReadFile(knownHosts) + if err != nil { + t.Fatalf("read known_hosts: %v", err) + } + if !strings.Contains(string(data), "example.com") { + t.Fatalf("known_hosts = %q, want example.com entry", data) + } + if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil { + t.Fatalf("second callback() error = %v", err) + } +} + +func TestAcceptNewHostKeyCallbackRejectsChangedHostKey(t *testing.T) { + first := testPublicKey(t) + second := testPublicKey(t) + knownHosts := filepath.Join(t.TempDir(), "known_hosts") + if err := os.WriteFile(knownHosts, []byte(knownhosts.Line([]string{knownhosts.Normalize("example.com:22")}, first)+"\n"), 0o600); err != nil { + t.Fatalf("write known_hosts: %v", err) + } + callback, err := acceptNewHostKeyCallback(Options{KnownHosts: knownHosts}) + if err != nil { + t.Fatalf("acceptNewHostKeyCallback() error = %v", err) + } + + err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, second) + if err == nil || !strings.Contains(err.Error(), "has changed") { + t.Fatalf("callback() error = %v, want changed host key", err) + } +} + +func TestAcceptNewHostKeyCallbackRequiresWritableKnownHostsForUnknownHost(t *testing.T) { + callback, err := acceptNewHostKeyCallback(Options{}) + if err != nil { + t.Fatalf("acceptNewHostKeyCallback() error = %v", err) + } + + err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, testPublicKey(t)) + if err == nil || !strings.Contains(err.Error(), "no writable known_hosts path") { + t.Fatalf("callback() error = %v, want no writable known_hosts path", err) + } +} + +func TestStrictHostKeyCallbackRequiresKnownHosts(t *testing.T) { + _, err := hostKeyCallback(Options{HostKeyPolicy: HostKeyPolicyStrict}) + if err == nil || !strings.Contains(err.Error(), "known_hosts is required") { + t.Fatalf("hostKeyCallback() error = %v, want known_hosts required", err) + } +} + +func testPublicKey(t *testing.T) cryptossh.PublicKey { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate key: %v", err) + } + publicKey, err := cryptossh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + t.Fatalf("new public key: %v", err) + } + return publicKey +} diff --git a/internal/adapters/ssh/integration_test.go b/internal/adapters/ssh/integration_test.go new file mode 100644 index 0000000..c4b5cce --- /dev/null +++ b/internal/adapters/ssh/integration_test.go @@ -0,0 +1,39 @@ +package ssh + +import ( + "context" + "os" + "strconv" + "testing" +) + +func TestIntegrationSSHBackendStatRoot(t *testing.T) { + host := os.Getenv("DISTRIBUTOR_TEST_SSH_HOST") + if host == "" { + t.Skip("DISTRIBUTOR_TEST_SSH_HOST is not set") + } + port := 22 + if raw := os.Getenv("DISTRIBUTOR_TEST_SSH_PORT"); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil { + t.Fatalf("parse DISTRIBUTOR_TEST_SSH_PORT: %v", err) + } + port = parsed + } + backend, err := New(context.Background(), Options{ + Host: host, + User: os.Getenv("DISTRIBUTOR_TEST_SSH_USER"), + Port: port, + Root: os.Getenv("DISTRIBUTOR_TEST_SSH_PATH"), + KeyFile: os.Getenv("DISTRIBUTOR_TEST_SSH_KEY_FILE"), + KnownHosts: os.Getenv("DISTRIBUTOR_TEST_SSH_KNOWN_HOSTS"), + HostKeyPolicy: HostKeyPolicyStrict, + }) + if err != nil { + t.Fatalf("New() error = %v", err) + } + defer backend.Close() + if _, err := backend.Stat(context.Background(), ""); err != nil { + t.Fatalf("Stat(root) error = %v", err) + } +} diff --git a/internal/adapters/ssh/options.go b/internal/adapters/ssh/options.go new file mode 100644 index 0000000..c95a2c9 --- /dev/null +++ b/internal/adapters/ssh/options.go @@ -0,0 +1,77 @@ +package ssh + +import ( + "fmt" + "os" + "os/user" + "path" + "path/filepath" + "strconv" +) + +const ( + BackendName = "ssh" + + HostKeyPolicyStrict HostKeyPolicy = "strict" + HostKeyPolicyAcceptNew HostKeyPolicy = "accept-new" + HostKeyPolicyOff HostKeyPolicy = "off" +) + +type HostKeyPolicy string + +type Options struct { + Host string + User string + Port int + Root string + KeyFile string + KnownHosts string + HostKeyPolicy HostKeyPolicy +} + +func (o Options) normalized() (Options, error) { + if o.Host == "" { + return Options{}, fmt.Errorf("host is required") + } + if o.User == "" { + current, err := user.Current() + if err != nil || current.Username == "" { + return Options{}, fmt.Errorf("user is required when current OS user cannot be determined") + } + o.User = current.Username + } + if o.Port == 0 { + o.Port = 22 + } + if o.Port < 1 || o.Port > 65535 { + return Options{}, fmt.Errorf("port must be between 1 and 65535") + } + if o.Root == "" { + return Options{}, fmt.Errorf("path is required") + } + o.Root = path.Clean(o.Root) + if o.HostKeyPolicy == "" { + o.HostKeyPolicy = HostKeyPolicyAcceptNew + } + switch o.HostKeyPolicy { + case HostKeyPolicyStrict, HostKeyPolicyAcceptNew, HostKeyPolicyOff: + default: + return Options{}, fmt.Errorf("host_key_policy must be strict, accept-new, or off") + } + if o.KnownHosts == "" && o.HostKeyPolicy != HostKeyPolicyOff { + o.KnownHosts = defaultKnownHostsPath() + } + return o, nil +} + +func (o Options) address() string { + return o.Host + ":" + strconv.Itoa(o.Port) +} + +func defaultKnownHostsPath() string { + home, err := os.UserHomeDir() + if err != nil || home == "" { + return "" + } + return filepath.Join(home, ".ssh", "known_hosts") +} diff --git a/internal/adapters/ssh/options_test.go b/internal/adapters/ssh/options_test.go new file mode 100644 index 0000000..6fcf8f0 --- /dev/null +++ b/internal/adapters/ssh/options_test.go @@ -0,0 +1,118 @@ +package ssh + +import ( + "context" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/distributor/internal/storage" + "github.com/pkg/sftp" +) + +func TestOptionsNormalizeDefaultsUserPortAndHostKeyPolicy(t *testing.T) { + options, err := (Options{ + Host: "example.com", + Root: "/reports", + }).normalized() + if err != nil { + t.Fatalf("normalized() error = %v", err) + } + if options.User == "" { + t.Fatal("normalized user is empty") + } + if options.Port != 22 { + t.Fatalf("port = %d, want 22", options.Port) + } + if options.HostKeyPolicy != HostKeyPolicyAcceptNew { + t.Fatalf("host key policy = %q, want accept-new", options.HostKeyPolicy) + } +} + +func TestOptionsNormalizeRejectsInvalidFields(t *testing.T) { + tests := map[string]Options{ + "host": {Root: "/reports"}, + "port": { + Host: "example.com", + Port: 70000, + Root: "/reports", + }, + "path": { + Host: "example.com", + }, + "host key policy": { + Host: "example.com", + Root: "/reports", + HostKeyPolicy: "prompt", + }, + } + for name, options := range tests { + t.Run(name, func(t *testing.T) { + if _, err := options.normalized(); err == nil { + t.Fatal("normalized() error = nil, want error") + } + }) + } +} + +func TestNativePathEnforcesLogicalPathRules(t *testing.T) { + backend := &Backend{root: "/srv/reports"} + tests := map[string]string{ + "bundle/report.md": "/srv/reports/bundle/report.md", + "": "/srv/reports", + } + for logicalPath, want := range tests { + t.Run(logicalPath, func(t *testing.T) { + got, err := backend.nativePath(logicalPath, true) + if err != nil { + t.Fatalf("nativePath() error = %v", err) + } + if got != want { + t.Fatalf("nativePath() = %q, want %q", got, want) + } + }) + } + + for _, logicalPath := range []string{"/absolute", "../escape", "a/../b", `a\b`} { + t.Run("reject "+logicalPath, func(t *testing.T) { + _, err := backend.nativePath(logicalPath, true) + if err == nil || !storage.IsInvalidPath(err) { + t.Fatalf("nativePath() error = %v, want invalid path", err) + } + }) + } +} + +func TestNewRejectsMissingAuthBeforeDial(t *testing.T) { + t.Setenv("SSH_AUTH_SOCK", "") + _, err := New(context.Background(), Options{ + Host: "example.com", + User: "reports", + Root: "/reports", + HostKeyPolicy: HostKeyPolicyOff, + }) + if err == nil || !strings.Contains(err.Error(), "no SSH auth methods configured") { + t.Fatalf("New() error = %v, want missing auth", err) + } +} + +func TestTranslateErrorMapsSFTPStatusCodes(t *testing.T) { + backend := &Backend{} + tests := []struct { + name string + err error + want storage.ErrorKind + }{ + {name: "not found", err: sftp.ErrSSHFxNoSuchFile, want: storage.ErrNotFound}, + {name: "permission", err: sftp.ErrSSHFxPermissionDenied, want: storage.ErrPermission}, + {name: "unsupported", err: sftp.ErrSSHFxOpUnsupported, want: storage.ErrUnsupported}, + {name: "temporary", err: sftp.ErrSSHFxConnectionLost, want: storage.ErrTemporary}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := backend.translateError(storage.OpStat, "report.md", tt.err) + if !storage.IsKind(err, tt.want) { + t.Fatalf("translateError() = %v, want kind %s", err, tt.want) + } + }) + } +} diff --git a/internal/app/backends.go b/internal/app/backends.go index 8717379..0991cf5 100644 --- a/internal/app/backends.go +++ b/internal/app/backends.go @@ -3,14 +3,25 @@ package app import ( "context" "fmt" + "strconv" "gitea.maximumdirect.net/eric/distributor/internal/adapters/local" + sshadapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/ssh" "gitea.maximumdirect.net/eric/distributor/internal/config" "gitea.maximumdirect.net/eric/distributor/internal/storage" ) const storagePathKey = "path" +const ( + sshHostKey = "host" + sshUserKey = "user" + sshPortKey = "port" + sshKeyFileKey = "ssh_key_file" + sshKnownHostsKey = "known_hosts" + sshHostKeyPolicyKey = "host_key_policy" +) + type backendFactory struct { registry *storage.Registry } @@ -23,23 +34,64 @@ func newBackendFactory() *backendFactory { } return local.New(cfg[storagePathKey]) }) + _ = registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) { + port, err := strconv.Atoi(cfg[sshPortKey]) + if err != nil { + return nil, fmt.Errorf("ssh port: %w", err) + } + return sshadapter.New(ctx, sshadapter.Options{ + Host: cfg[sshHostKey], + User: cfg[sshUserKey], + Port: port, + Root: cfg[storagePathKey], + KeyFile: cfg[sshKeyFileKey], + KnownHosts: cfg[sshKnownHostsKey], + HostKeyPolicy: sshadapter.HostKeyPolicy(cfg[sshHostKeyPolicyKey]), + }) + }) return &backendFactory{registry: registry} } func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) { - if source.Backend != config.BackendLocal { + if source.Backend != config.BackendLocal && source.Backend != config.BackendSSH { return nil, fmt.Errorf("source backend %s is not implemented for execution", source.Backend) } - return f.registry.Open(ctx, source.Backend, storage.OpenConfig{storagePathKey: source.Path}) + return f.registry.Open(ctx, source.Backend, sourceOpenConfig(source)) } func (f *backendFactory) openDestination(ctx context.Context, destination config.Destination) (storage.Backend, error) { - if destination.Backend != config.BackendLocal { + if destination.Backend != config.BackendLocal && destination.Backend != config.BackendSSH { return nil, fmt.Errorf("backend %s is not implemented for execution", destination.Backend) } - return f.registry.Open(ctx, destination.Backend, storage.OpenConfig{storagePathKey: destination.Path}) + return f.registry.Open(ctx, destination.Backend, destinationOpenConfig(destination)) } func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storage.Backend, error) { return f.registry.Open(ctx, config.BackendLocal, storage.OpenConfig{storagePathKey: path}) } + +func sourceOpenConfig(source config.Backend) storage.OpenConfig { + cfg := storage.OpenConfig{storagePathKey: source.Path} + if source.Backend == config.BackendSSH { + cfg[sshHostKey] = source.Host + cfg[sshUserKey] = source.User + cfg[sshPortKey] = strconv.Itoa(source.Port) + cfg[sshKeyFileKey] = source.SSH.KeyFile + cfg[sshKnownHostsKey] = source.SSH.KnownHosts + cfg[sshHostKeyPolicyKey] = string(source.SSH.HostKeyPolicy) + } + return cfg +} + +func destinationOpenConfig(destination config.Destination) storage.OpenConfig { + cfg := storage.OpenConfig{storagePathKey: destination.Path} + if destination.Backend == config.BackendSSH { + cfg[sshHostKey] = destination.Host + cfg[sshUserKey] = destination.User + cfg[sshPortKey] = strconv.Itoa(destination.Port) + cfg[sshKeyFileKey] = destination.SSH.KeyFile + cfg[sshKnownHostsKey] = destination.SSH.KnownHosts + cfg[sshHostKeyPolicyKey] = string(destination.SSH.HostKeyPolicy) + } + return cfg +} diff --git a/internal/app/backends_test.go b/internal/app/backends_test.go index ac42bc3..4ecd23a 100644 --- a/internal/app/backends_test.go +++ b/internal/app/backends_test.go @@ -6,6 +6,8 @@ import ( "testing" "gitea.maximumdirect.net/eric/distributor/internal/config" + "gitea.maximumdirect.net/eric/distributor/internal/storage" + "gitea.maximumdirect.net/eric/distributor/internal/storage/fake" ) func TestBackendFactoryOpensLocalSource(t *testing.T) { @@ -47,14 +49,72 @@ func TestBackendFactoryOpensDirectLocalPath(t *testing.T) { } } +func TestBackendFactoryOpensSSHSourceWithRegisteredOpener(t *testing.T) { + factory := &backendFactory{registry: storage.NewRegistry()} + var got storage.OpenConfig + if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) { + got = cfg + return fake.New(), nil + }); err != nil { + t.Fatalf("Register() error = %v", err) + } + + backend, err := factory.openSource(context.Background(), config.Backend{ + Backend: config.BackendSSH, + Host: "source.example.com", + User: "reports", + Port: 22, + Path: "/reports", + SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew}, + }) + if err != nil { + t.Fatalf("openSource() error = %v", err) + } + if backend == nil { + t.Fatal("openSource() backend = nil") + } + if got[sshHostKey] != "source.example.com" || got[storagePathKey] != "/reports" { + t.Fatalf("open config = %#v, want SSH source fields", got) + } +} + +func TestBackendFactoryOpensSSHDestinationWithRegisteredOpener(t *testing.T) { + factory := &backendFactory{registry: storage.NewRegistry()} + var got storage.OpenConfig + if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) { + got = cfg + return fake.New(), nil + }); err != nil { + t.Fatalf("Register() error = %v", err) + } + + backend, err := factory.openDestination(context.Background(), config.Destination{ + Backend: config.BackendSSH, + Host: "destination.example.com", + User: "deploy", + Port: 2222, + Path: "/archive", + SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyStrict}, + }) + if err != nil { + t.Fatalf("openDestination() error = %v", err) + } + if backend == nil { + t.Fatal("openDestination() backend = nil") + } + if got[sshHostKey] != "destination.example.com" || got[sshPortKey] != "2222" || got[sshHostKeyPolicyKey] != "strict" { + t.Fatalf("open config = %#v, want SSH destination fields", got) + } +} + func TestBackendFactoryRejectsUnsupportedSource(t *testing.T) { factory := newBackendFactory() _, err := factory.openSource(context.Background(), config.Backend{ - Backend: config.BackendSSH, - URI: "ssh://reports@example.com:22", - Path: "/reports", + Backend: config.BackendS3, + Endpoint: "https://s3.example.com", + Bucket: "reports", }) - if err == nil || !strings.Contains(err.Error(), "source backend ssh is not implemented for execution") { + if err == nil || !strings.Contains(err.Error(), "source backend s3 is not implemented for execution") { t.Fatalf("openSource() error = %v, want not implemented", err) } } @@ -70,3 +130,58 @@ func TestBackendFactoryRejectsUnsupportedDestination(t *testing.T) { t.Fatalf("openDestination() error = %v, want not implemented", err) } } + +func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) { + cfg := sourceOpenConfig(config.Backend{ + Backend: config.BackendSSH, + Host: "source.example.com", + User: "reports", + Port: 2222, + Path: "/reports", + SSH: config.SSH{ + KeyFile: "/home/reports/.ssh/id_ed25519", + KnownHosts: "/home/reports/.ssh/known_hosts", + HostKeyPolicy: config.HostKeyPolicyStrict, + }, + }) + + assertOpenConfig(t, cfg, map[string]string{ + storagePathKey: "/reports", + sshHostKey: "source.example.com", + sshUserKey: "reports", + sshPortKey: "2222", + sshKeyFileKey: "/home/reports/.ssh/id_ed25519", + sshKnownHostsKey: "/home/reports/.ssh/known_hosts", + sshHostKeyPolicyKey: "strict", + }) +} + +func TestBackendFactoryBuildsSSHDestinationOpenConfig(t *testing.T) { + cfg := destinationOpenConfig(config.Destination{ + Backend: config.BackendSSH, + Host: "destination.example.com", + User: "deploy", + Port: 22, + Path: "/srv/archive", + SSH: config.SSH{ + HostKeyPolicy: config.HostKeyPolicyAcceptNew, + }, + }) + + assertOpenConfig(t, cfg, map[string]string{ + storagePathKey: "/srv/archive", + sshHostKey: "destination.example.com", + sshUserKey: "deploy", + sshPortKey: "22", + sshHostKeyPolicyKey: "accept-new", + }) +} + +func assertOpenConfig(t *testing.T, got map[string]string, want map[string]string) { + t.Helper() + for key, wantValue := range want { + if gotValue := got[key]; gotValue != wantValue { + t.Fatalf("open config %s = %q, want %q", key, gotValue, wantValue) + } + } +} diff --git a/internal/app/run.go b/internal/app/run.go index 22fcd19..87916ca 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -52,16 +52,23 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error } } for _, pipeline := range cfg.Pipelines { + if options.Stdout != nil { + if err := writeSSHWarnings(options.Stdout, pipeline); err != nil { + return err + } + } sourceBackend, err := backends.openSource(ctx, pipeline.Source) if err != nil { return fmt.Errorf("pipeline %s: %w", pipeline.ID, err) } bundles, err := bundle.Discover(ctx, sourceBackend, "") if err != nil { + closeBackend(sourceBackend) return fmt.Errorf("pipeline %s discover source bundles: %w", pipeline.ID, err) } if options.Stdout != nil { if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil { + closeBackend(sourceBackend) return err } } @@ -76,6 +83,13 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error } continue } + closeDestination := true + deferCloseDestination := func() { + if closeDestination { + closeBackend(destinationBackend) + closeDestination = false + } + } req := publish.Request{ PipelineID: pipeline.ID, DestinationID: destination.ID, @@ -97,6 +111,7 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error writePlanLine(options.Stdout, plan, err) } if err != nil { + deferCloseDestination() failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err) summary.recordFailure() continue @@ -104,20 +119,24 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error summary.recordPlan(plan.Action) if !options.DryRun { if err := publish.Execute(ctx, req, plan); err != nil { + deferCloseDestination() failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err) summary.recordFailure() continue } if shouldNotify(plan.Action) { if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil { + deferCloseDestination() failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err) summary.recordFailure() continue } } } + deferCloseDestination() } } + closeBackend(sourceBackend) } if options.Stdout != nil { if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil { @@ -130,6 +149,18 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error return nil } +type closeableBackend interface { + Close() error +} + +func closeBackend(backend storage.Backend) { + closeable, ok := backend.(closeableBackend) + if !ok { + return + } + _ = closeable.Close() +} + func writePlanLine(w io.Writer, plan publish.Plan, planErr error) { if w == nil { return @@ -174,6 +205,22 @@ func destinationSummary(destinations []config.Destination) string { return strings.Join(ids, ",") } +func writeSSHWarnings(w io.Writer, pipeline config.Pipeline) error { + if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff { + if _, err := fmt.Fprintf(w, "Warning: pipeline=%s source host_key_policy=off disables SSH host key checking\n", pipeline.ID); err != nil { + return err + } + } + for _, destination := range pipeline.Destinations { + if destination.Backend == config.BackendSSH && destination.SSH.HostKeyPolicy == config.HostKeyPolicyOff { + if _, err := fmt.Fprintf(w, "Warning: pipeline=%s destination=%s host_key_policy=off disables SSH host key checking\n", pipeline.ID, destination.ID); err != nil { + return err + } + } + } + return nil +} + func shouldNotify(action publish.Action) bool { return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder } diff --git a/internal/app/run_test.go b/internal/app/run_test.go index 7db596f..7c16407 100644 --- a/internal/app/run_test.go +++ b/internal/app/run_test.go @@ -11,6 +11,7 @@ import ( "time" "gitea.maximumdirect.net/eric/distributor/internal/bundle" + "gitea.maximumdirect.net/eric/distributor/internal/config" "gitea.maximumdirect.net/eric/distributor/internal/notify" "gitea.maximumdirect.net/eric/distributor/internal/state" "gitea.maximumdirect.net/eric/distributor/internal/storage" @@ -46,6 +47,34 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) { } } +func TestWriteSSHWarningsReportsInsecureHostKeyPolicy(t *testing.T) { + var stdout bytes.Buffer + err := writeSSHWarnings(&stdout, config.Pipeline{ + ID: "reports", + Source: config.Backend{ + Backend: config.BackendSSH, + SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff}, + }, + Destinations: []config.Destination{{ + ID: "archive", + Backend: config.BackendSSH, + SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff}, + }}, + }) + if err != nil { + t.Fatalf("writeSSHWarnings() error = %v", err) + } + output := stdout.String() + for _, want := range []string{ + "pipeline=reports source host_key_policy=off disables SSH host key checking", + "pipeline=reports destination=archive host_key_policy=off disables SSH host key checking", + } { + if !strings.Contains(output, want) { + t.Fatalf("output = %q, want substring %q", output, want) + } + } +} + func TestRunPublishesNewLocalBundle(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() diff --git a/internal/config/config.go b/internal/config/config.go index 865f49a..b8ac924 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,6 +14,9 @@ type Pipeline struct { type Destination struct { ID string `yaml:"id"` Backend string `yaml:"backend"` + Host string `yaml:"host"` + User string `yaml:"user"` + Port int `yaml:"port"` Path string `yaml:"path"` URI string `yaml:"uri"` Endpoint string `yaml:"endpoint"` @@ -22,6 +25,7 @@ type Destination struct { Region string `yaml:"region"` ForcePath bool `yaml:"force_path_style"` Creds Credentials `yaml:"credentials"` + SSH SSH `yaml:",inline"` Publish *PublishPolicy `yaml:"publish"` Transform Transform `yaml:"transform"` Transfer TransferPolicy `yaml:"transfer"` @@ -29,6 +33,9 @@ type Destination struct { type Backend struct { Backend string `yaml:"backend"` + Host string `yaml:"host"` + User string `yaml:"user"` + Port int `yaml:"port"` Path string `yaml:"path"` URI string `yaml:"uri"` Endpoint string `yaml:"endpoint"` @@ -37,6 +44,13 @@ type Backend struct { Region string `yaml:"region"` ForcePath bool `yaml:"force_path_style"` Creds Credentials `yaml:"credentials"` + SSH SSH `yaml:",inline"` +} + +type SSH struct { + KeyFile string `yaml:"ssh_key_file"` + KnownHosts string `yaml:"known_hosts"` + HostKeyPolicy HostKeyPolicy `yaml:"host_key_policy"` } type Credentials struct { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 5a4c11d..287022d 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -25,11 +25,13 @@ const ( func ApplyDefaults(cfg *Config) { for pipelineIndex := range cfg.Pipelines { pipeline := &cfg.Pipelines[pipelineIndex] + applyBackendDefaults(&pipeline.Source) if pipeline.Validation.OnDigestMismatch == "" { pipeline.Validation.OnDigestMismatch = ValidationActionFail } for destinationIndex := range pipeline.Destinations { destination := &pipeline.Destinations[destinationIndex] + applyDestinationDefaults(destination) if destination.Publish == nil { destination.Publish = &PublishPolicy{Source: true} } @@ -48,3 +50,25 @@ func ApplyDefaults(cfg *Config) { } } } + +func applyBackendDefaults(backend *Backend) { + if backend.Backend == BackendSSH { + if backend.Port == 0 { + backend.Port = 22 + } + if backend.SSH.HostKeyPolicy == "" { + backend.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew + } + } +} + +func applyDestinationDefaults(destination *Destination) { + if destination.Backend == BackendSSH { + if destination.Port == 0 { + destination.Port = 22 + } + if destination.SSH.HostKeyPolicy == "" { + destination.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew + } + } +} diff --git a/internal/config/load_test.go b/internal/config/load_test.go index 470a8c6..d1e5709 100644 --- a/internal/config/load_test.go +++ b/internal/config/load_test.go @@ -53,7 +53,9 @@ pipelines: html: false - id: static-site backend: ssh - uri: ssh://deploy@example.com:22 + host: example.com + user: deploy + port: 22 path: /srv/www/reports publish: source: false @@ -87,13 +89,19 @@ pipelines: - id: ssh-backend source: backend: ssh - uri: ssh://reports@example.com:22 + host: source.example.com + user: reports path: /source destinations: - id: ssh-destination backend: ssh - uri: ssh://deploy@example.com:22 + host: destination.example.com + user: deploy + port: 2222 path: /destination + ssh_key_file: /home/deploy/.ssh/id_ed25519 + known_hosts: /home/deploy/.ssh/known_hosts + host_key_policy: strict `, "s3": ` pipelines: @@ -171,7 +179,7 @@ func TestLoadFileRejectsMissingRequiredFields(t *testing.T) { "destinations": `pipelines: [{id: reports, source: {backend: local, path: /source}}]`, "destination id": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{backend: local, path: /archive}]}]`, "local path": `pipelines: [{id: reports, source: {backend: local}, destinations: [{id: archive, backend: local, path: /archive}]}]`, - "ssh uri": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`, + "ssh host": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`, "s3 bucket": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com"}, destinations: [{id: archive, backend: local, path: /archive}]}]`, "publish outputs": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, publish: {source: false, html: false}}]}]`, } @@ -183,6 +191,88 @@ func TestLoadFileRejectsMissingRequiredFields(t *testing.T) { } } +func TestLoadFileDefaultsSSHConfig(t *testing.T) { + cfg := loadConfig(t, ` +pipelines: + - id: ssh-defaults + source: + backend: ssh + host: source.example.com + path: /source + destinations: + - id: archive + backend: ssh + host: destination.example.com + path: /archive + host_key_policy: false +`) + + source := cfg.Pipelines[0].Source + if source.Port != 22 { + t.Fatalf("source port = %d, want 22", source.Port) + } + if source.SSH.HostKeyPolicy != HostKeyPolicyAcceptNew { + t.Fatalf("source host key policy = %q, want accept-new", source.SSH.HostKeyPolicy) + } + destination := cfg.Pipelines[0].Destinations[0] + if destination.Port != 22 { + t.Fatalf("destination port = %d, want 22", destination.Port) + } + if destination.SSH.HostKeyPolicy != HostKeyPolicyOff { + t.Fatalf("destination host key policy = %q, want off", destination.SSH.HostKeyPolicy) + } +} + +func TestLoadFileNormalizesSSHHostKeyPolicies(t *testing.T) { + tests := map[string]HostKeyPolicy{ + `true`: HostKeyPolicyStrict, + `"true"`: HostKeyPolicyStrict, + `strict`: HostKeyPolicyStrict, + `accept-new`: HostKeyPolicyAcceptNew, + `false`: HostKeyPolicyOff, + `"false"`: HostKeyPolicyOff, + `off`: HostKeyPolicyOff, + `"STRICT"`: HostKeyPolicyStrict, + `"ACCEPT-NEW"`: HostKeyPolicyAcceptNew, + `"OFF"`: HostKeyPolicyOff, + } + for value, want := range tests { + t.Run(value, func(t *testing.T) { + cfg := loadConfig(t, ` +pipelines: + - id: ssh-policy + source: + backend: ssh + host: source.example.com + path: /source + host_key_policy: `+value+` + destinations: + - id: archive + backend: local + path: /archive +`) + if got := cfg.Pipelines[0].Source.SSH.HostKeyPolicy; got != want { + t.Fatalf("host key policy = %q, want %q", got, want) + } + }) + } +} + +func TestLoadFileRejectsSSHURIExecutionConfig(t *testing.T) { + assertLoadError(t, ` +pipelines: + - id: reports + source: + backend: ssh + uri: ssh://reports@example.com:22 + path: /source + destinations: + - id: archive + backend: local + path: /archive +`, "uri is not supported for ssh backend") +} + func TestLoadFileRejectsUnsupportedBackend(t *testing.T) { assertLoadError(t, ` pipelines: @@ -267,6 +357,7 @@ func TestExampleConfigsLoad(t *testing.T) { "../../examples/local-publish.yml", "../../examples/local-html.yml", "../../examples/fan-out.yml", + "../../examples/ssh-destination.yml", } { t.Run(path, func(t *testing.T) { if _, err := LoadFile(path); err != nil { diff --git a/internal/config/ssh.go b/internal/config/ssh.go new file mode 100644 index 0000000..2ec9ab6 --- /dev/null +++ b/internal/config/ssh.go @@ -0,0 +1,64 @@ +package config + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +type HostKeyPolicy string + +const ( + HostKeyPolicyStrict HostKeyPolicy = "strict" + HostKeyPolicyAcceptNew HostKeyPolicy = "accept-new" + HostKeyPolicyOff HostKeyPolicy = "off" +) + +func (p *HostKeyPolicy) UnmarshalYAML(value *yaml.Node) error { + switch value.Kind { + case yaml.ScalarNode: + default: + return fmt.Errorf("host_key_policy must be a boolean or string") + } + + switch value.Tag { + case "!!bool": + var enabled bool + if err := value.Decode(&enabled); err != nil { + return err + } + if enabled { + *p = HostKeyPolicyStrict + } else { + *p = HostKeyPolicyOff + } + return nil + case "!!str": + var raw string + if err := value.Decode(&raw); err != nil { + return err + } + normalized, ok := NormalizeHostKeyPolicy(raw) + if !ok { + return fmt.Errorf("host_key_policy must be strict, true, accept-new, off, or false") + } + *p = normalized + return nil + default: + return fmt.Errorf("host_key_policy must be a boolean or string") + } +} + +func NormalizeHostKeyPolicy(value string) (HostKeyPolicy, bool) { + switch strings.ToLower(value) { + case "", string(HostKeyPolicyAcceptNew): + return HostKeyPolicyAcceptNew, true + case string(HostKeyPolicyStrict), "true": + return HostKeyPolicyStrict, true + case string(HostKeyPolicyOff), "false": + return HostKeyPolicyOff, true + default: + return "", false + } +} diff --git a/internal/config/validate.go b/internal/config/validate.go index d555858..bf35cd2 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -37,7 +37,7 @@ func Validate(cfg Config) error { pipelineIDs[pipeline.ID] = struct{}{} } - errs = validateBackend(errs, pipelineContext+".source", pipeline.Source.Backend, pipeline.Source.Path, pipeline.Source.URI, pipeline.Source.Endpoint, pipeline.Source.Bucket) + errs = validateSourceBackend(errs, pipelineContext+".source", pipeline.Source) errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation) if len(pipeline.Destinations) == 0 { errs = append(errs, pipelineContext+".destinations is required") @@ -56,7 +56,7 @@ func Validate(cfg Config) error { destinationIDs[destination.ID] = struct{}{} } - errs = validateBackend(errs, destinationContext, destination.Backend, destination.Path, destination.URI, destination.Endpoint, destination.Bucket) + errs = validateDestinationBackend(errs, destinationContext, destination) errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform) errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer) } @@ -68,7 +68,15 @@ func Validate(cfg Config) error { return nil } -func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoint, bucket string) ValidationErrors { +func validateSourceBackend(errs ValidationErrors, context string, backend Backend) ValidationErrors { + return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.URI, backend.Endpoint, backend.Bucket, backend.SSH.HostKeyPolicy) +} + +func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors { + return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.URI, destination.Endpoint, destination.Bucket, destination.SSH.HostKeyPolicy) +} + +func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, uri, endpoint, bucket string, hostKeyPolicy HostKeyPolicy) ValidationErrors { switch backend { case "": errs = append(errs, context+".backend is required") @@ -77,12 +85,26 @@ func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoin errs = append(errs, context+".path is required for local backend") } case BackendSSH: - if uri == "" { - errs = append(errs, context+".uri is required for ssh backend") + if host == "" { + errs = append(errs, context+".host is required for ssh backend") } if path == "" { errs = append(errs, context+".path is required for ssh backend") } + if uri != "" { + errs = append(errs, context+".uri is not supported for ssh backend; use host, user, port, and path") + } + if port < 0 || port > 65535 { + errs = append(errs, context+".port must be between 1 and 65535") + } + if port == 0 { + errs = append(errs, context+".port is required for ssh backend after defaults are applied") + } + if hostKeyPolicy != "" { + if _, ok := NormalizeHostKeyPolicy(string(hostKeyPolicy)); !ok { + errs = append(errs, context+".host_key_policy must be strict, true, accept-new, off, or false") + } + } case BackendS3: if endpoint == "" { errs = append(errs, context+".endpoint is required for s3 backend")