Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 529172c754 | |||
| 7eed1a26ae | |||
| 1d71a151cc | |||
| 01e408f4d5 | |||
| 48169dc8b4 | |||
| 7a174ce5f1 | |||
| 14fa9c8000 | |||
| 052aa8a64a | |||
| 84f77ec0d0 | |||
| 1ad566264f |
@@ -2,7 +2,7 @@
|
||||
|
||||
`distributor` validates manifested report bundles and publishes selected source or generated artifacts to configured destinations.
|
||||
|
||||
It is currently a local-first CLI: source bundles are read from local storage, destinations are local directories, and Markdown files can be rendered to HTML sidecars.
|
||||
It is a local-first CLI with SSH/SFTP and S3-compatible storage support: source bundles can be read from local or remote storage, destinations can be local directories or remote paths, and Markdown files can be rendered to HTML sidecars.
|
||||
|
||||
Run the local example pipeline:
|
||||
|
||||
|
||||
20
docs/cli.md
20
docs/cli.md
@@ -13,17 +13,17 @@ This discovers the example source bundle and publishes source files to `workspac
|
||||
```sh
|
||||
distributor [--help]
|
||||
distributor version
|
||||
distributor run [--config <path>] [--dry-run]
|
||||
distributor run [--config <path>] [--dry-run] [--force]
|
||||
distributor validate <path>
|
||||
distributor inspect <path>
|
||||
```
|
||||
|
||||
- `version`: prints the application name and version. Development builds print `distributor dev`.
|
||||
- `run`: loads a YAML config, discovers local source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary.
|
||||
- `run`: loads a YAML config, discovers source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary.
|
||||
- `validate`: validates a local source bundle directory or a local tree containing source bundles.
|
||||
- `inspect`: validates local source bundles and prints normalized bundle metadata.
|
||||
|
||||
`validate` and `inspect` accept local paths only. `run` currently executes local backends only. SSH and S3 config can be parsed and validated, but configured SSH or S3 execution fails with a clear unsupported-execution error.
|
||||
`validate` and `inspect` accept local paths only. `run` executes `local`, `ssh`, and `s3` backends.
|
||||
|
||||
## Flag reference
|
||||
|
||||
@@ -38,7 +38,8 @@ All subcommands:
|
||||
`run` flags:
|
||||
|
||||
- `--config <path>`: config file to load. If omitted, `run` uses `/usr/local/etc/distributor/config.yml`.
|
||||
- `--dry-run`: load config, discover bundles, inspect destination state, print planned actions and final status, and do not write files.
|
||||
- `--dry-run`: load config, discover bundles, inspect destination state, print planned actions and final status, and do not write output files, destination state, or SSH `known_hosts` entries.
|
||||
- `--force`: allow explicit destructive replacement for supported conflict cases in this run only.
|
||||
|
||||
`run` does not accept positional arguments. `validate` and `inspect` accept at most one path; omitting the path returns a required-path error.
|
||||
|
||||
@@ -80,18 +81,27 @@ Preview local fan-out publication:
|
||||
go run ./cmd/distributor run --config examples/fan-out.yml --dry-run
|
||||
```
|
||||
|
||||
Preview a forced replacement before publishing:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run --force
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
`run` prints the number of configured pipelines, one line per pipeline, one line per planned destination action, and a final status line. Actions include:
|
||||
`run` prints the number of configured pipelines, one line per pipeline, one line per planned destination action, and a final status line. Destination action lines include the bundle path, destination id, destination backend, action, outputs, and reason. Actions include:
|
||||
|
||||
- `publish_new`: destination has no managed state and is empty.
|
||||
- `replace_older`: destination state is older than the source manifest.
|
||||
- `force_replace`: `--force` requested a supported destructive replacement.
|
||||
- `skip_same`: destination state already matches the source manifest.
|
||||
- `skip_destination_newer`: destination state is newer than the source manifest.
|
||||
- `error`: planning or execution failed for that destination.
|
||||
|
||||
The command exits non-zero if any destination fails. Independent later destinations are still attempted.
|
||||
|
||||
The final status line includes counters for `publish_new`, `replace_older`, `force_replace`, skipped destinations, failures, and whether the run was a dry run.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Use `validate` before publication when a producer has written a new bundle. Use `inspect` to confirm normalized ids, timestamps, digests, file paths, and file sizes.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Distributor Configuration
|
||||
# Configuration Reference
|
||||
|
||||
## Config File Location
|
||||
|
||||
@@ -10,7 +10,7 @@ If `--config` is omitted, `run` uses:
|
||||
/usr/local/etc/distributor/config.yml
|
||||
```
|
||||
|
||||
Config parsing rejects unknown YAML fields. The current executable backend support is local only. SSH and S3 config fields are accepted by config validation, but runtime execution for those backends is unavailable.
|
||||
Config parsing rejects unknown YAML fields. The executable backends are `local`, `ssh`, and `s3`.
|
||||
|
||||
## Minimal Local Config
|
||||
|
||||
@@ -72,6 +72,7 @@ Sidecar generation writes `report.html` for `report.md`. It does not mutate the
|
||||
|
||||
Top level:
|
||||
|
||||
- `secrets.directory`: optional credential secrets directory.
|
||||
- `pipelines`: required non-empty list.
|
||||
|
||||
Pipeline:
|
||||
@@ -85,12 +86,17 @@ Source backend:
|
||||
|
||||
- `backend`: required.
|
||||
- `path`: required for `local` and `ssh`.
|
||||
- `uri`: required for `ssh`.
|
||||
- `host`: required for `ssh`.
|
||||
- `user`: optional for `ssh`; defaults to the current OS user when available.
|
||||
- `port`: optional for `ssh`; defaults to `22`.
|
||||
- `ssh_key_file`: optional for `ssh`.
|
||||
- `known_hosts`: optional for `ssh`; defaults to the service user's OpenSSH `known_hosts` path when available.
|
||||
- `host_key_policy`: optional for `ssh`; defaults to `accept-new`.
|
||||
- `endpoint`: required for `s3`.
|
||||
- `bucket`: required for `s3`.
|
||||
- `prefix`: optional for `s3`.
|
||||
- `region`: optional for `s3`.
|
||||
- `force_path_style`: optional for `s3`.
|
||||
- `prefix`: optional for `s3`; leading and trailing slashes are trimmed.
|
||||
- `region`: optional for `s3`; defaults to `us-east-1`.
|
||||
- `force_path_style`: optional for `s3`; defaults to `true`. Set `false` only for services that require virtual-host addressing.
|
||||
- `credentials.access_key_id_env`: optional S3 credential environment variable name.
|
||||
- `credentials.secret_access_key_env`: optional S3 credential environment variable name.
|
||||
|
||||
@@ -105,8 +111,53 @@ Destination:
|
||||
Accepted backend names:
|
||||
|
||||
- `local`: executable; requires `path`.
|
||||
- `ssh`: config validation only; execution is unavailable.
|
||||
- `s3`: config validation only; execution is unavailable.
|
||||
- `ssh`: executable; requires `host` and `path`.
|
||||
- `s3`: executable; requires `endpoint` and `bucket`.
|
||||
|
||||
## SSH Backend
|
||||
|
||||
SSH uses native SFTP. It can be used for sources, destinations, or both:
|
||||
|
||||
```yaml
|
||||
backend: ssh
|
||||
host: example.com
|
||||
user: distributor
|
||||
port: 2222
|
||||
path: /remote/root
|
||||
ssh_key_file: /home/distributor/.ssh/id_ed25519
|
||||
known_hosts: /home/distributor/.ssh/known_hosts
|
||||
host_key_policy: accept-new
|
||||
```
|
||||
|
||||
Authentication uses SSH agent identities first when `SSH_AUTH_SOCK` is set, then `ssh_key_file` if configured. Password authentication in YAML is not supported.
|
||||
|
||||
Host key policies:
|
||||
|
||||
- `strict`, `true`, and `"true"` require a matching known host key.
|
||||
- `accept-new` accepts and persists a new host key, but fails if an existing key changed. During `run --dry-run`, new host keys are accepted only for the current connection and are not persisted.
|
||||
- `off`, `false`, and `"false"` disable host key checking and are insecure.
|
||||
|
||||
`accept-new` and `strict` use `known_hosts` when configured. If omitted, distributor uses the current service user's default OpenSSH `known_hosts` path where practical. `accept-new` fails when it needs to persist a new host key and no writable `known_hosts` path is available. It does not create a missing parent `.ssh` directory.
|
||||
|
||||
## S3 Backend
|
||||
|
||||
S3 uses the AWS SDK for Go v2 and supports S3-compatible endpoints:
|
||||
|
||||
```yaml
|
||||
backend: s3
|
||||
endpoint: https://s3.example.com
|
||||
bucket: reports
|
||||
prefix: archive
|
||||
region: us-east-1
|
||||
force_path_style: true
|
||||
credentials:
|
||||
access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
|
||||
secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
|
||||
```
|
||||
|
||||
`endpoint` and `bucket` are required. `prefix` is an optional backend root; it is treated as an object-key prefix, not a real directory. Prefixes must be clean slash-separated paths after trimming leading and trailing slashes. `http://` endpoints are allowed for explicitly configured local development or local S3-compatible test services.
|
||||
|
||||
If either credential environment variable name is configured, both must be configured and both referenced variables must resolve to non-empty values through the real process environment or `secrets.directory`. Explicit credentials take precedence over the AWS SDK default credential chain. If credential environment variable names are omitted, the SDK default credential chain is used and `secrets.directory` values are not injected into the process environment.
|
||||
|
||||
Publish policy:
|
||||
|
||||
@@ -119,14 +170,20 @@ Transfer policy:
|
||||
|
||||
- `transfer.on_destination_same`: `skip` or `fail`; defaults to `skip`.
|
||||
- `transfer.on_destination_older`: `replace` or `fail`; defaults to `replace`.
|
||||
- `transfer.on_destination_newer`: `skip` or `fail`; defaults to `skip`.
|
||||
- `transfer.on_conflict`: only `fail`; defaults to `fail`.
|
||||
- `transfer.on_destination_newer`: `skip`, `replace`, or `fail`; defaults to `skip`.
|
||||
- `transfer.on_conflict`: `fail` or `replace`; defaults to `fail`.
|
||||
|
||||
`replace` for `on_destination_newer` and `on_conflict` is honored only when `run --force` is used for that invocation. Force is CLI-only; there is no persistent config field that enables forced replacement by default.
|
||||
|
||||
## Defaults
|
||||
|
||||
Defaults are applied after YAML decoding and before validation:
|
||||
|
||||
- `validation.on_digest_mismatch: fail`
|
||||
- SSH `port: 22`
|
||||
- SSH `host_key_policy: accept-new`
|
||||
- S3 `region: us-east-1`
|
||||
- S3 `force_path_style: true`
|
||||
- `publish.source: true`
|
||||
- `publish.html: false`
|
||||
- `transfer.on_destination_same: skip`
|
||||
@@ -136,13 +193,22 @@ Defaults are applied after YAML decoding and before validation:
|
||||
|
||||
## Secrets
|
||||
|
||||
Do not put literal secrets in config files. S3 credentials may name environment variables:
|
||||
Do not put literal secrets in config files. `secrets.directory` lets deployments provide credential values as files:
|
||||
|
||||
```yaml
|
||||
secrets:
|
||||
directory: /run/secrets/distributor
|
||||
```
|
||||
|
||||
Each regular file in the directory becomes an internal credential environment value named by the filename. Valid filenames must match `[A-Za-z_][A-Za-z0-9_]*`. Directories are ignored, and symlinks to regular files are followed. Exactly one trailing LF or CRLF is trimmed from each file; other whitespace is preserved.
|
||||
|
||||
The resolver checks the real process environment first, then the secrets directory. If both define the same variable with different values, `run` prints a warning with the variable name and uses the real environment value. Secret values are not printed. The process environment is not modified, so SDK default credential chains see only real environment variables.
|
||||
|
||||
S3 credentials may name environment variables:
|
||||
|
||||
- `credentials.access_key_id_env`
|
||||
- `credentials.secret_access_key_env`
|
||||
|
||||
S3 execution is unavailable; these fields are accepted so config shape can be validated.
|
||||
|
||||
## Examples
|
||||
|
||||
Maintained examples live under [examples](../examples/):
|
||||
@@ -151,3 +217,5 @@ Maintained examples live under [examples](../examples/):
|
||||
- `local-publish.yml`: runnable local source publication.
|
||||
- `local-html.yml`: runnable local HTML publication.
|
||||
- `fan-out.yml`: runnable local fan-out publication to source and HTML destinations.
|
||||
- `ssh-destination.yml`: environment-gated local-to-SSH publication example.
|
||||
- `s3-destination.yml`: environment-gated local-to-S3 publication example.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
## Inputs and outputs
|
||||
|
||||
`Run` accepts a context, optional config path, dry-run flag, stdout writer, and optional notifier. It loads YAML config, discovers source bundles for each configured pipeline, plans each destination independently, optionally executes publish plans, writes summary output when stdout is supplied, and returns an aggregated error if any destination fails.
|
||||
`Run` accepts a context, optional config path, dry-run flag, force flag, stdout writer, and optional notifier. It loads YAML config, discovers source bundles for each configured pipeline, plans each destination independently, optionally executes publish plans, writes summary output when stdout is supplied, and returns an aggregated error if any destination fails.
|
||||
|
||||
`Validate` and `Inspect` accept a local path. `Validate` discovers and validates bundles. `Inspect` writes bundle metadata and manifest file entries to stdout when provided.
|
||||
|
||||
@@ -27,25 +27,27 @@ Destination failures are collected while later destinations continue to run. Sou
|
||||
|
||||
## Backend and transform wiring
|
||||
|
||||
The app-level backend factory registers only the local backend for execution. Config validation accepts other backend shapes, but `Run` can execute only local sources and local destinations.
|
||||
The app-level backend factory registers local, SSH, and S3 backends for execution. S3 explicit credential references are resolved through the config environment resolver before adapter construction.
|
||||
|
||||
The app-level transform registry registers Markdown-to-HTML using `internal/transform/markdown`. Lower-level publish code receives a resolver and does not import concrete transform implementations.
|
||||
|
||||
## Dry-run behavior
|
||||
|
||||
Dry-run still loads config, opens backends, discovers bundles, inspects destinations, resolves transforms, and builds publish plans. It does not write destination outputs, write `.distributor.json`, delete managed outputs, or notify.
|
||||
Dry-run still loads config, opens backends, discovers bundles, inspects destinations, resolves transforms, and builds publish plans. It does not write destination outputs, write `.distributor.json`, delete managed outputs, perform forced prefix deletion, or notify.
|
||||
|
||||
## Failure behavior
|
||||
|
||||
`Run` returns immediately for config loading errors, context cancellation before work starts, source open errors, and source discovery errors. Per-destination backend, planning, execution, and notification errors are aggregated into one run error after remaining destinations have been attempted.
|
||||
|
||||
Run diagnostics include pipeline id, destination id, destination backend, and bundle path for destination-scoped failures. Source open and discovery failures include the source backend.
|
||||
|
||||
Stdout write errors are returned immediately because the caller's requested output stream can no longer be trusted.
|
||||
|
||||
## Boundaries
|
||||
|
||||
`internal/app` coordinates packages but does not own manifest validation rules, destination state comparison, storage path rules, output planning, transform rendering, or backend-specific filesystem behavior.
|
||||
|
||||
`Validate` and `Inspect` are local path commands. Remote execution wiring is outside current behavior.
|
||||
`Validate` and `Inspect` are local path commands. Remote execution wiring currently belongs to `Run`.
|
||||
|
||||
## Tests
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
## Inputs and outputs
|
||||
|
||||
Input is a YAML file containing `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
|
||||
Input is a YAML file containing optional `secrets` and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
|
||||
|
||||
## Loading flow
|
||||
|
||||
@@ -14,11 +14,15 @@ Input is a YAML file containing `pipelines`. Output is a `Config` value with def
|
||||
|
||||
Known-field checking rejects misspelled or unknown YAML keys before defaults and validation run.
|
||||
|
||||
`LoadFile` does not read secret files. `Run` loads the configured secrets directory after config validation and before backend construction.
|
||||
|
||||
## Defaults
|
||||
|
||||
Defaults are applied in `ApplyDefaults`:
|
||||
|
||||
- pipeline validation defaults `on_digest_mismatch` to `fail`;
|
||||
- SSH backend `port` defaults to `22`;
|
||||
- SSH backend `host_key_policy` defaults to `accept-new`;
|
||||
- destination publish policy defaults to source output only;
|
||||
- `transfer.on_destination_same` defaults to `skip`;
|
||||
- `transfer.on_destination_older` defaults to `replace`;
|
||||
@@ -29,11 +33,25 @@ Defaults are applied in `ApplyDefaults`:
|
||||
|
||||
Validation requires at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, and valid transfer actions.
|
||||
|
||||
Transfer validation accepts `replace` for `on_destination_newer` and `on_conflict`, but publish planning honors those destructive actions only when the current run explicitly requests force.
|
||||
|
||||
`ValidatePublishTransformPolicy` is shared with publish planning so destination policy combinations are checked consistently. Publishing HTML requires an enabled Markdown-to-HTML transform in `sidecar` mode. A publish policy must select source output, HTML output, or both.
|
||||
|
||||
## Executable support boundary
|
||||
|
||||
Config validation accepts `local`, `ssh`, and `s3` backend shapes so config files can be validated as schemas. Runtime execution currently opens only local backends through `internal/app`.
|
||||
Config validation accepts `local`, `ssh`, and `s3` backend shapes. Runtime execution opens all three through `internal/app`.
|
||||
|
||||
SSH config uses structured fields: `host`, optional `user`, optional `port`, `path`, optional `ssh_key_file`, optional `known_hosts`, and optional `host_key_policy`. `host_key_policy` accepts YAML booleans and strings and normalizes `true`/`strict`, `accept-new`, and `false`/`off`.
|
||||
|
||||
S3 config requires `endpoint` and `bucket`, normalizes optional `prefix`, defaults `region` to `us-east-1`, and defaults omitted `force_path_style` to `true` while preserving explicit `false`.
|
||||
|
||||
## Secrets and credential resolution
|
||||
|
||||
`secrets.directory` points to a directory of credential files. `LoadSecretEnvironment` reads regular files and symlinks to regular files, rejects invalid filenames, trims exactly one trailing LF or CRLF, and returns an `Environment` resolver plus conflict metadata.
|
||||
|
||||
The resolver checks the real process environment first and loaded secret values second. Differing process/secret conflicts are reported by variable name only. The resolver does not mutate `os.Environ`; default SDK credential chains continue to see only real process environment values.
|
||||
|
||||
Credential-consuming backend wiring should resolve explicit credential environment variable references through `Environment.ResolveCredentials` or the same resolver pattern instead of calling `os.Getenv` directly.
|
||||
|
||||
The user-facing configuration reference is `docs/config.md`; this file documents package behavior for maintainers.
|
||||
|
||||
|
||||
@@ -6,38 +6,39 @@
|
||||
|
||||
## Inputs and outputs
|
||||
|
||||
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, publish policy, transform policy, transformer resolver, transfer policy, destination bundle path, and existing destination state.
|
||||
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, publish policy, transform policy, transformer resolver, transfer policy, destination bundle path, existing destination state, and whether explicit force was requested for the current run.
|
||||
|
||||
Output is a plan with an action, reason, and selected source or generated outputs. Execution writes selected source files, generated files, and `.distributor.json` for publish or replacement actions.
|
||||
|
||||
## Actions
|
||||
|
||||
Supported actions are `publish_new`, `replace_older`, `skip_same`, `skip_destination_newer`, `fail_conflict`, and `fail_unmanaged`.
|
||||
Supported actions are `publish_new`, `replace_older`, `force_replace`, `skip_same`, `skip_destination_newer`, `fail_conflict`, and `fail_unmanaged`.
|
||||
|
||||
## Failure behavior
|
||||
|
||||
Planning fails when request fields are incomplete, publish and transform policies are invalid, selected outputs collide, HTML output is requested without Markdown inputs, destination state is invalid, destination content is unmanaged, or transfer policy maps the comparison outcome to failure.
|
||||
Planning fails when request fields are incomplete, publish and transform policies are invalid, selected outputs collide, HTML output is requested without Markdown inputs, destination state is invalid, destination content is unmanaged without force, or transfer policy maps the comparison outcome to failure.
|
||||
|
||||
Execution fails if a write, delete, state serialization, or context check fails. Outputs written during a failed publish attempt are cleaned up through managed deletion where possible.
|
||||
|
||||
## Boundaries
|
||||
|
||||
The current implementation publishes source files and Markdown-to-HTML sidecar outputs. Backend behavior is supplied through `internal/storage`; app runtime currently supplies local backends.
|
||||
The current implementation publishes source files and Markdown-to-HTML sidecar outputs. Backend behavior is supplied through `internal/storage`; app runtime currently supplies local, SSH, and S3 backends.
|
||||
|
||||
The package uses `internal/state` for destination comparison, `internal/storage` for IO, and the shared `internal/config` publish/transform policy helper for request validation. It resolves transforms through a narrow resolver supplied by the caller; concrete transform registration is owned by the app layer. It does not parse CLI flags or load config files.
|
||||
|
||||
## Safety
|
||||
|
||||
Replacement deletes only outputs recorded in existing destination state plus `.distributor.json`. Failed local writes trigger cleanup of outputs written during the failed attempt.
|
||||
Normal replacement deletes only outputs recorded in existing destination state plus `.distributor.json`. Forced replacement deletes the bounded destination bundle path before writing outputs and state. Failed writes trigger cleanup of outputs written during the failed attempt where practical.
|
||||
|
||||
## Tests
|
||||
|
||||
Before changing publish behavior, inspect tests under `internal/publish` and local run tests under `internal/app`.
|
||||
Before changing publish behavior, inspect tests under `internal/publish` and run tests under `internal/app`.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Publish planning is deterministic for the same source, destination state, policies, and transform outputs.
|
||||
- Replacement deletes only managed paths recorded in existing state plus `.distributor.json`.
|
||||
- Normal replacement deletes only managed paths recorded in existing state plus `.distributor.json`.
|
||||
- Forced replacement is explicit per run and deletes only within the destination bundle path.
|
||||
- Publish execution writes destination state after selected outputs are written.
|
||||
- Transform implementations are resolved through an interface supplied by the caller.
|
||||
- Unmanaged destination content is never overwritten.
|
||||
- Unmanaged destination content is overwritten only by explicit forced replacement.
|
||||
|
||||
@@ -33,7 +33,7 @@ Comparison outcomes cover absent destination state, unmanaged destination conten
|
||||
|
||||
## Failure behavior
|
||||
|
||||
Invalid JSON, invalid state schema, invalid embedded source manifests, unsafe output paths, unsupported output kinds, missing generated-output transform names, and mismatched pipeline or destination ids produce comparison outcomes that publish planning can turn into fail actions.
|
||||
Invalid JSON, invalid state schema, invalid embedded source manifests, unsafe output paths, unsupported output kinds, missing generated-output transform names, and mismatched pipeline or destination ids produce comparison outcomes that publish planning can turn into fail actions. Supported identity and source-manifest conflicts can become forced replacement only when publish planning receives explicit force and compatible transfer policy.
|
||||
|
||||
## Boundaries
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
## Inputs and outputs
|
||||
|
||||
The storage interface supports byte reads, stream reads, byte writes, stream writes, exact metadata lookup, traversal, destination emptiness checks, and guarded managed deletion.
|
||||
The storage interface supports byte reads, stream reads, byte writes, stream writes, exact metadata lookup, traversal, destination emptiness checks, guarded managed deletion, and bounded prefix deletion for explicit forced replacement.
|
||||
|
||||
Entries report a logical path, type, and size when available. Entry types are `file`, `directory`, `symlink`, and `other`.
|
||||
|
||||
@@ -14,7 +14,7 @@ Entries report a logical path, type, and size when available. Entry types are `f
|
||||
|
||||
Core packages should depend on `internal/storage`, not adapter packages. Adapter-specific path handling stays behind backend implementations.
|
||||
|
||||
The local adapter lives in `internal/adapters/local`. Runtime backend construction is wired through the app-level backend factory and storage registry. The fake backend lives in `internal/storage/fake` for tests and is not registered for runtime use.
|
||||
The local adapter lives in `internal/adapters/local`. The SSH/SFTP adapter lives in `internal/adapters/ssh`. The S3-compatible adapter lives in `internal/adapters/s3`. Runtime backend construction is wired through the app-level backend factory and storage registry. The fake backend lives in `internal/storage/fake` for tests and is not registered for runtime use.
|
||||
|
||||
## Paths
|
||||
|
||||
@@ -28,12 +28,18 @@ Backends may wrap implementation-specific errors, but callers should receive sto
|
||||
|
||||
## Deletion
|
||||
|
||||
Backends expose guarded managed deletion only. `DeleteManagedBundle` may delete listed managed outputs plus `.distributor.json`; it does not provide broad recursive deletion.
|
||||
`DeleteManagedBundle` may delete listed managed outputs plus `.distributor.json`.
|
||||
|
||||
## Local and fake backends
|
||||
`DeletePrefix` removes content at and below a logical prefix for explicit forced replacement. It must not delete above the requested prefix or above the configured backend root.
|
||||
|
||||
## Local, SSH, S3, and fake backends
|
||||
|
||||
The local adapter maps logical paths to a configured filesystem root and keeps adapter-specific path handling behind the storage interface.
|
||||
|
||||
The SSH adapter maps logical paths to a configured remote SFTP root. It uses native SSH and SFTP libraries, supports SSH agent and key-file authentication, applies host-key policies, rejects unsafe logical paths, reports symlink entries from `Lstat`, and limits deletion to managed targets or explicit bounded prefixes.
|
||||
|
||||
The S3 adapter maps logical paths to object keys below a configured bucket and optional prefix. It uses the AWS SDK for Go v2, treats prefixes as object trees, requires exact objects for `Stat`, paginates traversal, applies conservative overwrite checks with `HeadObject`, infers basic content types, and limits deletion to managed target objects or explicit bounded object-key prefixes.
|
||||
|
||||
The fake backend is an in-memory implementation for package tests. It is not registered for runtime use.
|
||||
|
||||
## Tests
|
||||
@@ -43,6 +49,8 @@ Before changing storage behavior, inspect tests under:
|
||||
- `internal/storage`
|
||||
- `internal/storage/fake`
|
||||
- `internal/adapters/local`
|
||||
- `internal/adapters/ssh`
|
||||
- `internal/adapters/s3`
|
||||
|
||||
## Invariants
|
||||
|
||||
@@ -50,4 +58,5 @@ Before changing storage behavior, inspect tests under:
|
||||
- Logical paths are slash-separated and confined to the backend root.
|
||||
- `storage.List` uses backend traversal and returns deterministic entries.
|
||||
- Managed deletion is limited to recorded outputs plus `.distributor.json`.
|
||||
- Prefix deletion is limited to the requested logical prefix.
|
||||
- Runtime backend registration is owned by `internal/app`.
|
||||
|
||||
@@ -32,13 +32,29 @@ Preview local fan-out publication:
|
||||
go run ./cmd/distributor run --config examples/fan-out.yml --dry-run
|
||||
```
|
||||
|
||||
Preview an environment-gated SSH destination config after editing it for an SSH/SFTP endpoint you control:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config examples/ssh-destination.yml --dry-run
|
||||
```
|
||||
|
||||
Preview an environment-gated S3 destination config after editing it for an S3-compatible endpoint and bucket you control:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config examples/s3-destination.yml --dry-run
|
||||
```
|
||||
|
||||
## Filesystem Layout
|
||||
|
||||
Source bundles are discovered beneath the configured local source root. Each bundle is a directory containing `manifest.json`.
|
||||
Source bundles are discovered beneath the configured source root. Each bundle is a directory containing `manifest.json`.
|
||||
|
||||
Destination bundle paths preserve the source bundle path relative to the source root. A source bundle at the source root publishes to the destination root. A source bundle under `daily/` publishes under `daily/` at each destination.
|
||||
|
||||
The maintained examples write under `workspace/`, which is ignored by Git.
|
||||
The maintained local examples write under `workspace/`, which is ignored by Git.
|
||||
|
||||
SSH backends use the configured remote `path` as the backend root. Source bundle discovery and destination bundle paths are relative to that root, using the same logical path rules as local storage.
|
||||
|
||||
S3 backends use the configured bucket plus optional `prefix` as the backend root. Source bundle discovery and destination bundle paths are relative to that object-key prefix. Prefixes are object-key prefixes, not real directories.
|
||||
|
||||
## Destination State
|
||||
|
||||
@@ -56,9 +72,11 @@ Do not edit `.distributor.json` by hand during normal operation. If it is missin
|
||||
|
||||
## Dry Runs
|
||||
|
||||
`--dry-run` loads and validates config, discovers source bundles, inspects destination state, plans outputs, and prints summary lines. It does not write output files or destination state.
|
||||
`--dry-run` loads and validates config, discovers source bundles, inspects destination state, plans outputs, and prints summary lines. It does not write output files, destination state, or SSH `known_hosts` entries.
|
||||
|
||||
Dry-run output is useful before publishing to confirm actions such as `publish_new`, `replace_older`, `skip_same`, and `skip_destination_newer`.
|
||||
Dry-run output is useful before publishing to confirm actions such as `publish_new`, `replace_older`, `force_replace`, `skip_same`, and `skip_destination_newer`.
|
||||
|
||||
Destination action lines include the destination backend, so mixed local, SSH, and S3 fan-out runs can be audited before publication.
|
||||
|
||||
## Retry and Replacement Behavior
|
||||
|
||||
@@ -68,18 +86,64 @@ If destination state is older than the source manifest and transfer policy allow
|
||||
|
||||
If destination state is newer than the source manifest, the default behavior is to skip. If destination state has the same source id and created timestamp but a different digest, publication fails as a conflict.
|
||||
|
||||
If a destination path has files but no valid `.distributor.json`, publication fails as unmanaged content. There is no force overwrite option.
|
||||
If a destination path has files but no valid `.distributor.json`, publication fails as unmanaged content unless the current run explicitly uses `--force`.
|
||||
|
||||
## Force Workflow
|
||||
|
||||
Use `--force` only after a dry run shows the intended `force_replace` action:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run --force
|
||||
go run ./cmd/distributor run --config <config-path> --force
|
||||
```
|
||||
|
||||
Forced replacement can overwrite unmanaged non-empty destination paths. Destination state conflicts require `transfer.on_conflict: replace` plus `--force`. Newer destination state requires `transfer.on_destination_newer: replace` plus `--force`.
|
||||
|
||||
Forced replacement deletes the current destination bundle path before writing outputs and state. It does not delete above that bundle path. For S3 destinations, deletion is constrained to the configured bucket and prefix plus the destination bundle prefix. Force is per run only and has no config default.
|
||||
|
||||
## Failure Handling
|
||||
|
||||
If one destination fails in a fan-out run, independent later destinations are still planned and executed. The command exits non-zero after printing the final status if any destination failed.
|
||||
|
||||
If a write fails during local publication, `distributor` attempts to remove outputs written during that failed attempt so a retry does not see those partial outputs as unmanaged destination content.
|
||||
Errors include the pipeline id, destination id, destination backend, and bundle path where applicable.
|
||||
|
||||
If a write fails during publication, `distributor` attempts to remove outputs written during that failed attempt so a retry does not see those partial outputs as unmanaged destination content.
|
||||
|
||||
After a successful publish or replacement, the internal notifier hook runs. The current default notifier is a no-op. Skipped destinations do not invoke it.
|
||||
|
||||
## SSH Operation Notes
|
||||
|
||||
SSH execution uses SFTP over `golang.org/x/crypto/ssh` and `github.com/pkg/sftp`. It does not shell out to `ssh`, `scp`, or `rsync`.
|
||||
|
||||
Configure `ssh_key_file`, an SSH agent, or both. Agent identities are attempted first, followed by the configured key file. YAML password authentication is not supported.
|
||||
|
||||
The default host key policy is `accept-new`. New host keys are written to `known_hosts` when the file path is writable. During `--dry-run`, unknown host keys may be accepted for the current connection but are not written to `known_hosts`; a later non-dry-run may persist the same key. Changed host keys are fatal for both `strict` and `accept-new`. The `off` policy disables host key checking and `run` prints a warning when stdout is enabled.
|
||||
|
||||
Recovery boundaries are the same as local storage: replacement deletes only managed output paths recorded in `.distributor.json` plus the state file, and failed writes are cleaned up where practical. Distributor never performs broad recursive remote deletion.
|
||||
|
||||
## S3 Operation Notes
|
||||
|
||||
S3 execution uses the AWS SDK for Go v2. Configure `endpoint`, `bucket`, optional `prefix`, optional `region`, and optional explicit credential environment variable names.
|
||||
|
||||
When explicit credential env names are configured, both variables must resolve to non-empty values through the real process environment or `secrets.directory`. When they are omitted, the AWS SDK default credential chain is used as-is.
|
||||
|
||||
Normal replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Forced replacement deletes objects under the bounded destination bundle prefix. Distributor does not manage bucket versioning or delete markers.
|
||||
|
||||
## Secrets Directory
|
||||
|
||||
Configure `secrets.directory` when credential values should come from mounted files, such as deployment secrets:
|
||||
|
||||
```yaml
|
||||
secrets:
|
||||
directory: /run/secrets/distributor
|
||||
```
|
||||
|
||||
The directory is loaded during `run` before any source or destination backend is opened. If the directory is missing, unreadable, or contains an invalid secret filename, the run fails before publication work starts.
|
||||
|
||||
Real process environment values take precedence over files with the same name. If the values differ and stdout is enabled, `run` prints a warning naming the ignored secret file variable without printing either value. The process environment is not changed.
|
||||
|
||||
## Caveats
|
||||
|
||||
Only local-to-local execution is available. SSH execution, S3 execution, external notification adapters, and force overwrite behavior are unavailable.
|
||||
External notification adapters are unavailable. Force overwrite behavior is available only through the explicit `run --force` workflow.
|
||||
|
||||
For symptom-oriented fixes, see [troubleshooting](troubleshooting.md). For config details, see [configuration](config.md). For command syntax, see [CLI](cli.md).
|
||||
|
||||
@@ -166,7 +166,7 @@ For example, one destination may publish source files only as a long-term archiv
|
||||
|
||||
## Backend Abstraction
|
||||
|
||||
Sources and destinations use the same storage abstraction. Current runtime execution uses the local filesystem backend. Additional storage backends should be peer implementations behind the same interface, and any backend-specific execution limitation must be documented.
|
||||
Sources and destinations use the same storage abstraction. Current runtime execution uses the local filesystem, SSH/SFTP, and S3-compatible backends. Additional storage backends should be peer implementations behind the same interface, and any backend-specific execution limitation must be documented.
|
||||
|
||||
Application logic must interact with storage through internal backend interfaces. Backend-specific behavior belongs in adapter packages. Pipeline, bundle, state, publish, and transform packages must not import service-specific or filesystem adapter implementation details.
|
||||
|
||||
@@ -194,6 +194,8 @@ Use this current layout unless the project has a documented reason to differ:
|
||||
- `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
|
||||
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
|
||||
- `internal/adapters/local`: local filesystem backend.
|
||||
- `internal/adapters/ssh`: SSH/SFTP backend.
|
||||
- `internal/adapters/s3`: S3-compatible object storage backend.
|
||||
- `internal/transform`: transform interfaces, registry, planning, and shared transform models.
|
||||
- `internal/transform/markdown`: Markdown-to-HTML implementation.
|
||||
- `internal/publish`: destination planning, reconciliation, safety checks, and publish execution.
|
||||
@@ -272,9 +274,11 @@ If the application writes durable state, writes should be atomic where practical
|
||||
|
||||
Code that deletes, moves, or overwrites files must use narrow, explicit paths. Avoid broad parent-directory operations. Cleanup that can cause data loss must be opt-in.
|
||||
|
||||
`distributor` must never perform broad deletion against a configured source root or destination root. Destructive replacement may occur only inside a resolved destination bundle path when a valid `.distributor.json` confirms that the path is distributor-managed.
|
||||
`distributor` must never perform broad deletion against a configured source root. Destination deletion must be bounded to the resolved destination bundle path for the current source bundle and backend root.
|
||||
|
||||
Replacement must be narrow, logged, test-covered, and configurable. Prefer deleting files recorded in `.distributor.json` and known generated outputs rather than blindly deleting parent directories. Backend implementations must guard against path traversal, prefix confusion, and accidental root deletion.
|
||||
Normal destructive replacement may occur only when a valid `.distributor.json` confirms that the destination bundle path is distributor-managed. Explicit forced replacement is a per-run CLI workflow for supported conflict and unmanaged-content cases; it must be dry-runnable, clearly reported, and constrained to the destination bundle path.
|
||||
|
||||
Replacement must be narrow, reported, test-covered, and configurable. Prefer normal replacement that deletes files recorded in `.distributor.json` and known generated outputs. Forced replacement may delete a bounded destination bundle prefix only when the operator explicitly requests it. Backend implementations must guard against path traversal, prefix confusion, and accidental deletion above the configured backend root.
|
||||
|
||||
Where practical, publish operations should use staging paths or temporary objects and promote them into place only after validation and transform steps succeed.
|
||||
|
||||
|
||||
@@ -13,8 +13,10 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
|
||||
- `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
|
||||
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
|
||||
- `internal/adapters/local`: local filesystem backend.
|
||||
- `internal/adapters/ssh`: SSH/SFTP backend.
|
||||
- `internal/adapters/s3`: S3-compatible object storage backend.
|
||||
- `internal/storage/fake`: in-memory backend for tests.
|
||||
- `internal/publish`: destination inspection, output planning, reconciliation, execution, and managed cleanup.
|
||||
- `internal/publish`: destination inspection, output planning, reconciliation, execution, managed cleanup, and explicit forced replacement.
|
||||
- `internal/transform`: transform interface and registry.
|
||||
- `internal/transform/markdown`: Markdown-to-HTML sidecar transform.
|
||||
- `internal/notify`: notification interface and current no-op notifier.
|
||||
@@ -69,7 +71,7 @@ GOCACHE=/private/tmp/distributor-gocache GOMODCACHE=/private/tmp/distributor-gom
|
||||
- Prefer package-local helpers over broad abstractions until behavior is shared by multiple packages.
|
||||
- Keep CLI parsing in `internal/cli`; business decisions belong in `internal/app`, `internal/bundle`, `internal/publish`, `internal/state`, and related core packages.
|
||||
- Keep adapter packages thin. Backend-specific filesystem or service behavior belongs in adapters; bundle, state, transform, and publish policy belongs outside adapters.
|
||||
- Preserve public CLI behavior, config semantics, manifest schema, destination state schema, and local MVP behavior unless the current task explicitly changes them.
|
||||
- Preserve public CLI behavior, config semantics, manifest schema, destination state schema, and implemented backend behavior unless the current task explicitly changes them.
|
||||
- Use `storage.DisplayPath`, `storage.StateFileName`, `storage.StatePath`, and `storage.ManagedBundleTargets` instead of duplicating those conventions.
|
||||
- Use `bundle.ValidateManifest` for normalized source manifest semantics, including embedded source manifests in destination state.
|
||||
- Use `config.ValidatePublishTransformPolicy` for publish and transform policy combinations.
|
||||
@@ -82,6 +84,9 @@ The project currently depends on:
|
||||
|
||||
- `gopkg.in/yaml.v3` for YAML configuration loading.
|
||||
- `github.com/yuin/goldmark` for Markdown rendering.
|
||||
- `golang.org/x/crypto/ssh`, `golang.org/x/crypto/ssh/agent`, and `golang.org/x/crypto/ssh/knownhosts` for native SSH support.
|
||||
- `github.com/pkg/sftp` for native SFTP support.
|
||||
- `github.com/aws/aws-sdk-go-v2/...` packages for S3-compatible storage support.
|
||||
|
||||
Add external dependencies only when they materially improve correctness,
|
||||
security, interoperability, or implementation complexity. Avoid dependencies
|
||||
@@ -100,9 +105,15 @@ When adding or changing configuration:
|
||||
5. Update `docs/config.md` in the same change if current user-visible config behavior changes.
|
||||
6. Update examples only with configs that are valid and executable for implemented behavior.
|
||||
|
||||
Config validation may accept fields for backends that are not executable yet,
|
||||
but user-facing docs and examples must clearly state execution support. At the
|
||||
time of this policy, only the local backend is executable.
|
||||
Config validation may accept fields for roadmap backends before execution
|
||||
support exists, but user-facing docs and examples must clearly state execution
|
||||
support. Runtime executable backends are local, SSH, and S3.
|
||||
|
||||
Credential-consuming code must use the config-owned environment resolver for
|
||||
explicit credential environment variable references. Do not call `os.Getenv`
|
||||
directly for backend credentials, because `secrets.directory` values are
|
||||
intentionally available through the resolver without mutating the process
|
||||
environment.
|
||||
|
||||
## CLI Changes
|
||||
|
||||
@@ -117,7 +128,7 @@ When adding or changing commands or flags:
|
||||
4. Update `docs/cli.md` if syntax, flags, output expectations, or workflows change.
|
||||
|
||||
`validate` and `inspect` are local path commands. `run` loads configured
|
||||
pipelines and currently executes local backends only.
|
||||
pipelines and executes local, SSH, and S3 backends.
|
||||
|
||||
## Storage Backends
|
||||
|
||||
@@ -133,8 +144,8 @@ When adding a backend:
|
||||
5. Add focused adapter tests and app-level wiring tests.
|
||||
6. Update user docs, operations docs, examples, and internal docs only for behavior that is actually implemented.
|
||||
|
||||
Do not document SSH/SFTP or S3 execution as available until corresponding
|
||||
adapter packages and app wiring exist.
|
||||
Do not document future backend execution as available until the corresponding
|
||||
adapter package and app wiring exist.
|
||||
|
||||
## Transforms
|
||||
|
||||
@@ -158,16 +169,29 @@ Test close to the behavior being changed:
|
||||
- Use `internal/testutil` for shared valid fixtures only; keep edge cases near the package under test.
|
||||
- Run `go test ./...` after cross-package changes or documentation/example changes tied to tests.
|
||||
|
||||
Live integration tests must be opt-in and skipped during normal `go test ./...`
|
||||
unless their required environment variables are set. Test-only environment
|
||||
variables must use this prefix shape:
|
||||
|
||||
```text
|
||||
DISTRIBUTOR_TEST_<BACKEND>_*
|
||||
```
|
||||
|
||||
Examples include `DISTRIBUTOR_TEST_SSH_HOST` and
|
||||
`DISTRIBUTOR_TEST_S3_ENDPOINT`. Do not use production credential variable names
|
||||
for test-only controls.
|
||||
|
||||
## Examples
|
||||
|
||||
Examples under `examples/` must be valid, maintained, and free of secrets.
|
||||
They should be copyable for implemented behavior. Do not leave examples that
|
||||
look runnable but require unsupported backend execution.
|
||||
They should be copyable for implemented behavior. Remote examples must use
|
||||
placeholders or environment variables for endpoint and credential material.
|
||||
|
||||
When changing examples:
|
||||
|
||||
1. Keep paths relative to the repository where practical.
|
||||
2. Prefer local examples until remote backend support exists.
|
||||
2. Keep local examples runnable without external services; gate remote examples
|
||||
behind user-provided endpoints and credentials.
|
||||
3. Run `go test ./internal/config` because config tests load examples.
|
||||
4. Update README, CLI, or config docs if links or recommended workflows change.
|
||||
|
||||
|
||||
@@ -1,463 +1,62 @@
|
||||
# Post-Local-MVP Implementation Roadmap
|
||||
# Implementation Roadmap
|
||||
|
||||
This is the canonical active roadmap for `distributor` after the local MVP checkpoint.
|
||||
This roadmap records current implementation status and deferred work for
|
||||
`distributor`. Implemented behavior is documented in the user, operator, and
|
||||
internal documentation listed below.
|
||||
|
||||
The original MVP stages 1-8 are complete and are no longer listed as pending implementation work. Current behavior is documented outside the roadmap in `README.md`, `docs/cli.md`, `docs/config.md`, `docs/operations.md`, `docs/troubleshooting.md`, `docs/internal/`, `docs/integrations/markdown.md`, and `docs/policy/development.md`.
|
||||
Canonical current-behavior docs:
|
||||
|
||||
Future, planned, or aspirational behavior should remain under `docs/roadmap/` until implemented.
|
||||
- `README.md`
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/internal/`
|
||||
- `docs/integrations/markdown.md`
|
||||
- `docs/policy/`
|
||||
- `examples/`
|
||||
|
||||
## Current Baseline
|
||||
Future, planned, or aspirational behavior belongs under `docs/roadmap/` until
|
||||
it is implemented.
|
||||
|
||||
The implemented local MVP includes:
|
||||
## Current State
|
||||
|
||||
- standard-library CLI commands for `version`, `run`, `validate`, and `inspect`;
|
||||
- YAML config loading, defaults, known-field rejection, and validation;
|
||||
- accepted config shapes for `local`, `ssh`, and `s3`, with executable backend support currently limited to `local`;
|
||||
- backend-rooted storage interface with typed errors, safe logical paths, traversal, `HasAny`, managed deletion, local backend, and fake backend;
|
||||
- source bundle discovery, manifest parsing, RFC3339 timestamp handling, duplicate path checks, path safety checks, symlink rejection, per-file digest validation, and bundle digest validation;
|
||||
- destination `.distributor.json` state parsing, validation, output metadata, and source comparison;
|
||||
- local publication of source files, Markdown sidecar HTML, or both;
|
||||
- destination output collision detection before writes;
|
||||
- managed replacement for older destination state;
|
||||
- unmanaged destination and conflict failures by default;
|
||||
- deterministic dry-run output and final run summaries;
|
||||
- deterministic sequential fan-out with aggregated failures;
|
||||
- cleanup of outputs written during failed local publish attempts where practical;
|
||||
- no-op notification hook after successful publish or replacement;
|
||||
- current user, operator, internal, integration, and development documentation for implemented behavior.
|
||||
`distributor` is ready for routine use against producer pipelines using the
|
||||
implemented local, SSH/SFTP, and S3-compatible backends.
|
||||
|
||||
The local MVP intentionally does not include executable SSH/SFTP backends, executable S3-compatible backends, force overwrite behavior, external notification adapters, warning-only digest mismatch behavior, or broad recursive destination deletion.
|
||||
## Active Roadmap
|
||||
|
||||
## Active Roadmap Stages
|
||||
There are no active implementation items in this roadmap.
|
||||
|
||||
Implement each stage independently. Unless a stage explicitly says otherwise:
|
||||
|
||||
1. read `docs/policy/architecture.md`, `docs/policy/documentation.md`, `docs/policy/development.md`, and this roadmap before editing;
|
||||
2. preserve current local MVP behavior;
|
||||
3. keep user-facing docs limited to implemented behavior;
|
||||
4. add or update focused tests for the behavior changed;
|
||||
5. run the relevant package tests and `go test ./...` for cross-package changes;
|
||||
6. avoid implementing later stages early.
|
||||
|
||||
## Stage 1: SSH/SFTP Backend
|
||||
|
||||
### Goal
|
||||
|
||||
Implement native SSH/SFTP storage backend support for sources and destinations through the existing storage interface and app-level backend factory.
|
||||
|
||||
### Implementation Scope
|
||||
|
||||
Add an SSH/SFTP adapter package under `internal/adapters/ssh`.
|
||||
|
||||
The backend must implement the current `internal/storage.Backend` contract:
|
||||
|
||||
- `ReadFile` and `OpenReader`;
|
||||
- `WriteFile` and `WriteFrom`;
|
||||
- `Stat`;
|
||||
- `Walk`;
|
||||
- `HasAny`;
|
||||
- `DeleteManagedBundle`.
|
||||
|
||||
Use native SFTP operations rather than shelling out to `ssh`, `scp`, or `rsync`.
|
||||
|
||||
Authentication behavior:
|
||||
|
||||
- prefer SSH agent by default;
|
||||
- use `known_hosts` validation by default where practical;
|
||||
- support optional key-file configuration only if it can be added cleanly;
|
||||
- do not support passwords in YAML in this stage.
|
||||
|
||||
Config execution behavior:
|
||||
|
||||
- use the existing accepted config shape:
|
||||
|
||||
```yaml
|
||||
backend: ssh
|
||||
uri: ssh://user@example.com:22
|
||||
path: /remote/root
|
||||
```
|
||||
|
||||
- keep secrets out of config files;
|
||||
- keep config loading and validation centralized in `internal/config`;
|
||||
- wire runtime construction through app-level backend factory and storage registry patterns.
|
||||
|
||||
Supported pipeline combinations:
|
||||
|
||||
- local source to SSH destination;
|
||||
- SSH source to local destination;
|
||||
- SSH source to SSH destination where feasible through streaming or backend-owned staging.
|
||||
|
||||
Safety requirements:
|
||||
|
||||
- enforce the same backend-rooted logical path rules as local storage;
|
||||
- reject path traversal, absolute logical paths, dot segments, and backslashes;
|
||||
- report or reject symlinks according to storage and bundle validation rules;
|
||||
- keep deletion limited to managed output paths and `.distributor.json`;
|
||||
- never delete a configured backend root;
|
||||
- preserve conservative non-force conflict behavior.
|
||||
|
||||
### Documentation Updates
|
||||
|
||||
After implementation, update only current-behavior docs:
|
||||
|
||||
- `docs/config.md`: mark SSH as executable and document any implemented SSH-only fields.
|
||||
- `docs/operations.md`: add SSH source/destination operating notes and recovery boundaries.
|
||||
- `docs/troubleshooting.md`: add common SSH authentication, known-hosts, and remote path failures.
|
||||
- `docs/internal/storage.md`: add implemented SSH adapter behavior and tests.
|
||||
- `docs/policy/development.md`: update backend addition guidance if implementation changes the pattern.
|
||||
- `examples/`: add only runnable or clearly environment-gated SSH examples.
|
||||
|
||||
Do not document S3 or force overwrite as implemented in this stage.
|
||||
|
||||
### Tests
|
||||
|
||||
Add unit tests for:
|
||||
|
||||
- SSH config execution wiring;
|
||||
- URI and path handling;
|
||||
- logical path validation;
|
||||
- storage error translation where practical;
|
||||
- `Walk` and `HasAny` behavior through test doubles or controlled fixtures;
|
||||
- managed deletion boundaries;
|
||||
- app-level local-to-SSH and SSH-to-local planning or wiring using fakes/mocks where possible.
|
||||
|
||||
Add integration tests only if they are skipped unless explicit SSH test endpoint environment variables are configured. Normal `go test ./...` must not require a live SSH server.
|
||||
|
||||
### Completion Criteria
|
||||
|
||||
- SSH/SFTP backend compiles and satisfies `storage.Backend`.
|
||||
- Runtime `run` can execute supported SSH source and destination flows.
|
||||
- Local MVP tests still pass.
|
||||
- Normal test runs do not require a live SSH server.
|
||||
- User docs accurately describe implemented SSH behavior and boundaries.
|
||||
|
||||
## Stage 2: S3-Compatible Backend
|
||||
|
||||
### Goal
|
||||
|
||||
Implement S3-compatible object storage backend support for sources and destinations through the existing storage interface and app-level backend factory.
|
||||
|
||||
### Implementation Scope
|
||||
|
||||
Add an S3-compatible adapter package under `internal/adapters/s3`.
|
||||
|
||||
The backend must implement the current `internal/storage.Backend` contract:
|
||||
|
||||
- `ReadFile` and `OpenReader`;
|
||||
- `WriteFile` and `WriteFrom`;
|
||||
- `Stat`;
|
||||
- `Walk`;
|
||||
- `HasAny`;
|
||||
- `DeleteManagedBundle`.
|
||||
|
||||
Use the existing accepted config shape:
|
||||
|
||||
```yaml
|
||||
backend: s3
|
||||
endpoint: https://s3.example.com
|
||||
bucket: reports
|
||||
prefix: some/prefix
|
||||
region: us-east-1
|
||||
force_path_style: true
|
||||
credentials:
|
||||
access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
|
||||
secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
|
||||
```
|
||||
|
||||
Credential behavior:
|
||||
|
||||
- read access key and secret key from the named environment variables when configured;
|
||||
- support standard SDK credential discovery only if it does not weaken explicit config behavior;
|
||||
- do not put literal secrets in YAML.
|
||||
|
||||
Object semantics:
|
||||
|
||||
- treat prefixes as object trees, not real directories;
|
||||
- normalize configured prefix plus logical path with exact path-boundary matching;
|
||||
- `Stat` must not synthesize directory metadata only because objects exist below a prefix;
|
||||
- `Walk` should use object-list pagination and should not load an entire prefix into memory;
|
||||
- `HasAny` should stop after the first matching object;
|
||||
- `DeleteManagedBundle` must delete only listed managed output objects plus `.distributor.json`.
|
||||
|
||||
Write behavior:
|
||||
|
||||
- treat successful object PUT as publish-on-success;
|
||||
- set content type from `storage.WriteOptions` where available;
|
||||
- spool or buffer `WriteFrom` only when required by the SDK or backend;
|
||||
- preserve overwrite checks and conservative conflict behavior.
|
||||
|
||||
Content type behavior should cover at least:
|
||||
|
||||
- `.md`: `text/markdown; charset=utf-8`;
|
||||
- `.html`: `text/html; charset=utf-8`;
|
||||
- `.json`: `application/json`;
|
||||
- `.txt`: `text/plain; charset=utf-8`.
|
||||
|
||||
Supported pipeline combinations:
|
||||
|
||||
- local source to S3 destination;
|
||||
- S3 source to local destination;
|
||||
- S3 source to S3 destination where feasible through streaming or backend-owned staging.
|
||||
|
||||
### Documentation Updates
|
||||
|
||||
After implementation, update only current-behavior docs:
|
||||
|
||||
- `docs/config.md`: mark S3 as executable and document implemented credential behavior.
|
||||
- `docs/operations.md`: add S3 source/destination layout, prefix, and recovery notes.
|
||||
- `docs/troubleshooting.md`: add common S3 credential, bucket, endpoint, prefix, and permission failures.
|
||||
- `docs/internal/storage.md`: add implemented S3 adapter behavior and tests.
|
||||
- `docs/policy/development.md`: update backend addition guidance if implementation changes the pattern.
|
||||
- `examples/`: add only safe S3 examples that use placeholder endpoints and environment variable names.
|
||||
|
||||
Do not document force overwrite or notification adapters as implemented in this stage.
|
||||
|
||||
### Tests
|
||||
|
||||
Add unit tests for:
|
||||
|
||||
- S3 config execution wiring;
|
||||
- credential environment variable handling;
|
||||
- key and prefix normalization;
|
||||
- exact prefix boundary behavior;
|
||||
- path traversal rejection;
|
||||
- content type selection;
|
||||
- paginated `Walk` behavior through mocks/fakes;
|
||||
- early-stop `HasAny`;
|
||||
- managed deletion boundaries;
|
||||
- publish planning with S3 destination state fixtures.
|
||||
|
||||
Add integration tests only if they are skipped unless explicit S3-compatible endpoint credentials are configured. Normal `go test ./...` must not require live S3 credentials.
|
||||
|
||||
### Completion Criteria
|
||||
|
||||
- S3-compatible backend compiles and satisfies `storage.Backend`.
|
||||
- Runtime `run` can execute supported S3 source and destination flows.
|
||||
- Local and SSH behavior, if implemented, remain unchanged.
|
||||
- Normal test runs do not require live S3.
|
||||
- User docs accurately describe implemented S3 behavior and boundaries.
|
||||
|
||||
## Stage 3: Cross-Backend Hardening and Documentation
|
||||
|
||||
### Goal
|
||||
|
||||
Harden behavior across implemented backend combinations, improve operator-facing failures, and synchronize current-behavior documentation and examples after remote backend support exists.
|
||||
|
||||
### Implementation Scope
|
||||
|
||||
Exercise and harden representative flows across all implemented backend types:
|
||||
|
||||
- local source to local archive destination;
|
||||
- local source to local HTML destination;
|
||||
- local source to multiple destinations with different publish policies;
|
||||
- local source to SSH destination, when SSH is implemented and test credentials exist;
|
||||
- SSH source to local destination, when SSH is implemented and test credentials exist;
|
||||
- local source to S3 destination, when S3 is implemented and test credentials exist;
|
||||
- S3 source to local destination, when S3 is implemented and test credentials exist.
|
||||
|
||||
Improve error context where practical for:
|
||||
|
||||
- invalid config;
|
||||
- invalid source manifest;
|
||||
- digest mismatch;
|
||||
- destination conflict;
|
||||
- unmanaged destination path;
|
||||
- backend read, write, stat, walk, and delete failures;
|
||||
- transform failures;
|
||||
- partial fan-out failures.
|
||||
|
||||
Ensure errors and logs identify pipeline id, destination id, bundle path or id, backend type, and logical path where useful without exposing secrets.
|
||||
|
||||
Do not add force overwrite behavior in this stage.
|
||||
|
||||
### Documentation Updates
|
||||
|
||||
Update current-behavior docs after hardening:
|
||||
|
||||
- `README.md`: keep the quickstart local unless remote examples become safe and concise.
|
||||
- `docs/cli.md`: document any changed output or diagnostics.
|
||||
- `docs/config.md`: ensure backend support status and config reference match implementation.
|
||||
- `docs/operations.md`: document cross-backend state layout, retry behavior, and recovery caveats.
|
||||
- `docs/troubleshooting.md`: add recurring SSH/S3 failure modes discovered during hardening.
|
||||
- `docs/internal/`: update storage, publish, app, and config internals where behavior changed.
|
||||
- `examples/`: keep examples copyable and free of secrets; remote examples must rely on placeholders and environment variables.
|
||||
|
||||
### Tests
|
||||
|
||||
Add or expand tests for:
|
||||
|
||||
- dry-run across multiple destinations and backend types;
|
||||
- partial failure behavior;
|
||||
- repeated run idempotency;
|
||||
- older/newer destination state behavior across backends;
|
||||
- destination state output metadata accuracy;
|
||||
- generated HTML output metadata accuracy;
|
||||
- destructive replacement safety across implemented backends;
|
||||
- error context for common failures.
|
||||
|
||||
Integration tests for SSH or S3 must remain opt-in through environment variables.
|
||||
|
||||
### Completion Criteria
|
||||
|
||||
- Implemented backend combinations behave consistently through the common pipeline path.
|
||||
- Repeated runs are idempotent.
|
||||
- Destructive paths remain bounded to managed destination bundle paths.
|
||||
- Operator-facing errors are actionable.
|
||||
- Current-behavior docs and examples match implemented backend support.
|
||||
|
||||
## Stage 4: Explicit Force Overwrite
|
||||
|
||||
### Goal
|
||||
|
||||
Introduce explicit operator-requested force behavior for controlled overwrite cases that remain intentionally unsupported by default.
|
||||
|
||||
### Implementation Scope
|
||||
|
||||
Add a CLI-only force option:
|
||||
|
||||
```bash
|
||||
distributor run --config config.yml --force
|
||||
```
|
||||
|
||||
Force must be explicit per run. Do not add a persistent config default for force behavior.
|
||||
|
||||
Define and implement force planning for:
|
||||
|
||||
- unmanaged non-empty destination paths;
|
||||
- destination state with a different source id;
|
||||
- destination state with matching source id and matching `created` timestamp but different digest;
|
||||
- destination state with mismatched `pipeline_id` or `destination_id`;
|
||||
- destination newer than source when transfer policy explicitly allows replacement.
|
||||
|
||||
Once force behavior exists, update transfer policy validation only for values supported by implemented force behavior:
|
||||
|
||||
- `on_destination_newer: replace`;
|
||||
- `on_conflict: replace`.
|
||||
|
||||
Safety requirements:
|
||||
|
||||
- non-force behavior remains unchanged and conservative;
|
||||
- dry-run must show destructive force actions before any forced run;
|
||||
- force must never delete above the resolved destination bundle path or configured destination prefix;
|
||||
- local replacement should remain staged where practical;
|
||||
- S3 replacement must remain constrained to the destination bundle prefix;
|
||||
- managed state should still be written only after successful output writes;
|
||||
- logs and output must clearly mark force decisions.
|
||||
|
||||
### Documentation Updates
|
||||
|
||||
After implementation, update:
|
||||
|
||||
- `docs/cli.md`: document `--force` syntax and dry-run workflow.
|
||||
- `docs/config.md`: document newly accepted transfer policy values and note force is CLI-only.
|
||||
- `docs/operations.md`: document safe force workflow and recovery boundaries.
|
||||
- `docs/troubleshooting.md`: describe when force may be appropriate and when it remains unsafe.
|
||||
- `docs/internal/publish.md` and `docs/internal/state.md`: document force planning and comparison handling.
|
||||
|
||||
Do not document force as a default or config-only behavior.
|
||||
|
||||
### Tests
|
||||
|
||||
Add tests for:
|
||||
|
||||
- force rejected or unavailable when the flag is absent;
|
||||
- unmanaged non-empty destination overwritten only with force;
|
||||
- different source id overwritten only with force and allowed policy;
|
||||
- same id and created timestamp with different digest overwritten only with force and allowed policy;
|
||||
- destination newer replaced only with force and allowed policy;
|
||||
- pipeline or destination id mismatch overwritten only with force and allowed policy;
|
||||
- dry-run reports destructive force actions without writing;
|
||||
- force deletes only bounded destination bundle paths;
|
||||
- local, SSH, and S3 backends, where implemented, preserve deletion boundaries.
|
||||
|
||||
### Completion Criteria
|
||||
|
||||
- Force overwrite behavior is explicit, logged, dry-runnable, and test-covered.
|
||||
- Default non-force behavior remains unchanged.
|
||||
- User docs clearly describe force risks and safe workflow.
|
||||
|
||||
## Stage 5: Release Readiness
|
||||
|
||||
### Goal
|
||||
|
||||
Perform a final quality pass before treating `distributor` as ready for routine use against real producer pipelines and implemented destination backends.
|
||||
|
||||
### Implementation Scope
|
||||
|
||||
Review:
|
||||
|
||||
- package boundaries against `docs/policy/architecture.md`;
|
||||
- contributor workflow against `docs/policy/development.md`;
|
||||
- user docs against `docs/policy/documentation.md`;
|
||||
- CLI UX and command output;
|
||||
- config validation and examples;
|
||||
- manifest and state compatibility;
|
||||
- destructive operation safety;
|
||||
- backend error handling;
|
||||
- logging and diagnostics for unattended operation;
|
||||
- test coverage for core invariants.
|
||||
|
||||
Do not add new product features in this stage.
|
||||
|
||||
### Documentation Updates
|
||||
|
||||
Update current-behavior docs only for issues found during the readiness review.
|
||||
|
||||
If release packaging, version injection, or installation workflow is added, document it in the appropriate current-behavior user or development docs.
|
||||
|
||||
### Tests
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Also verify representative CLI examples that are documented as runnable.
|
||||
|
||||
### Completion Criteria
|
||||
|
||||
- A dry-run can be performed safely against real configured sources and destinations.
|
||||
- Repeated runs are idempotent.
|
||||
- Destructive replacement cannot occur outside managed destination bundle paths.
|
||||
- Current docs accurately reflect the application.
|
||||
- The project is ready to deploy against one real producer pipeline.
|
||||
Before implementing new product behavior, add a focused roadmap entry when the
|
||||
work changes storage semantics, config, CLI behavior, state schema, transform
|
||||
behavior, notification behavior, operational safety, or user-visible workflows.
|
||||
Keep those entries out of current-behavior docs until the behavior exists.
|
||||
|
||||
## Deferred Work
|
||||
|
||||
The following work remains intentionally deferred unless a future roadmap promotes it:
|
||||
These items are not implemented and should stay out of current-behavior docs
|
||||
until a roadmap entry is selected and implemented:
|
||||
|
||||
- external notification adapters such as email, ntfy, Gotify, or Pushover;
|
||||
- RSS or Atom feed generation;
|
||||
- static site index pages beyond sidecar HTML output;
|
||||
- destination path remapping rules;
|
||||
- HTML themes beyond the minimal deterministic template;
|
||||
- full plugin architecture;
|
||||
- web UI;
|
||||
- report editing;
|
||||
- producer pipeline execution;
|
||||
- database-backed state;
|
||||
- complex retry queues;
|
||||
- concurrent publication workers;
|
||||
- symlink support;
|
||||
- external notification adapters;
|
||||
- warning-only digest mismatch handling;
|
||||
- password-based SSH authentication in YAML;
|
||||
- broad recursive or prefix deletion outside explicitly bounded force behavior.
|
||||
- additional auth mechanisms beyond the implemented SSH and S3 credential
|
||||
paths;
|
||||
- compatibility parsing for legacy SSH URI config;
|
||||
- broad recursive destination deletion outside managed bundle paths;
|
||||
- concurrent fan-out publishing;
|
||||
- streaming, resumable, or multipart S3 uploads;
|
||||
- cloud-provider-specific IAM integration docs;
|
||||
- repository-managed packaging, release, and deployment automation.
|
||||
|
||||
## Validation
|
||||
## Roadmap Maintenance
|
||||
|
||||
For roadmap-only edits:
|
||||
When adding future roadmap work:
|
||||
|
||||
```bash
|
||||
git status --short
|
||||
git diff -- docs/roadmap
|
||||
rg -n "docs/roadmap/(packages|contracts|storage|config|documentation)\\.md" README.md docs examples
|
||||
rg -n "docs/roadmap/(packages|contracts|storage|config|documentation)\\.md" .
|
||||
rg -n "SSH|S3|--force|force overwrite|notification adapter|future|planned" README.md docs/*.md docs/internal docs/policy examples
|
||||
```
|
||||
|
||||
Also search `docs/roadmap` for old MVP stage headings and titles from deleted roadmap files. That check should return no matches.
|
||||
|
||||
The final SSH/S3/force/future-work search is not expected to return zero results. Review matches and confirm they are either under roadmap material or clearly marked as unsupported current behavior.
|
||||
|
||||
Go tests are not required for documentation-only roadmap rationalization unless examples, behavior docs, or code change.
|
||||
- describe user-visible behavior and safety boundaries;
|
||||
- define which existing docs must change after implementation;
|
||||
- keep examples secret-free and runnable or clearly environment-gated;
|
||||
- avoid workflow labels in production code, tests, config fields, and user
|
||||
documentation;
|
||||
- run focused tests for the changed behavior and `go test ./...` for
|
||||
cross-package changes.
|
||||
|
||||
@@ -34,11 +34,11 @@ Diagnostic:
|
||||
rg -n "backend:" <config-path>
|
||||
```
|
||||
|
||||
Safe fix: use `backend: local` for executable workflows. SSH and S3 config shapes are accepted only for validation; runtime execution is unavailable.
|
||||
Safe fix: use `backend: local`, `backend: ssh`, or `backend: s3` for executable workflows.
|
||||
|
||||
## `backend ssh is not implemented for execution` or `backend s3 is not implemented for execution`
|
||||
## `prefix must be a clean relative slash-separated path`
|
||||
|
||||
Likely cause: the config validates but `run` tried to execute a remote backend.
|
||||
Likely cause: S3 `prefix` contains traversal, dot segments, empty segments, or backslashes after leading and trailing slashes are trimmed.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -46,7 +46,169 @@ Diagnostic:
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: use local destinations for current executable workflows, or keep remote backend configs under roadmap material unless those adapters are added. See [configuration](config.md).
|
||||
Safe fix: use a clean relative prefix such as `reports/archive`, or omit `prefix`.
|
||||
|
||||
## `NoSuchBucket`, `InvalidBucketName`, or `not_found`
|
||||
|
||||
Likely cause: the S3 bucket, endpoint, or prefix is wrong, or the configured credentials cannot see the requested object.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: verify `endpoint`, `bucket`, `region`, `force_path_style`, and `prefix`. For S3-compatible services, keep `force_path_style: true` unless the service requires virtual-host addressing.
|
||||
|
||||
## `AccessDenied`, `InvalidAccessKeyId`, or `SignatureDoesNotMatch`
|
||||
|
||||
Likely cause: S3 credentials are missing, wrong, empty, or lack permission for the bucket or prefix.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
env | cut -d= -f1 | rg '^(<access-key-variable>|<secret-key-variable>)$'
|
||||
ls -l <secrets-directory>
|
||||
```
|
||||
|
||||
Safe fix: provide both configured credential environment variables through the real environment or `secrets.directory`, or omit explicit credential fields to use the AWS SDK default credential chain.
|
||||
|
||||
## S3 endpoint connection failures
|
||||
|
||||
Likely cause: the endpoint URL is unreachable, uses the wrong scheme, or does not match the configured path-style mode.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
curl -I <endpoint>
|
||||
```
|
||||
|
||||
Safe fix: correct `endpoint`, network routing, TLS settings outside distributor, or `force_path_style`. Distributor does not provide insecure TLS bypass configuration.
|
||||
|
||||
## `load secrets directory ... no such file or directory`
|
||||
|
||||
Likely cause: `secrets.directory` points to a missing directory.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ls -ld <secrets-directory>
|
||||
```
|
||||
|
||||
Safe fix: create or mount the directory before running, or remove `secrets.directory` if no credential files are needed.
|
||||
|
||||
## `load secrets directory ... permission denied`
|
||||
|
||||
Likely cause: the service user cannot read the configured secrets directory.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ls -ld <secrets-directory>
|
||||
namei -l <secrets-directory>
|
||||
```
|
||||
|
||||
Safe fix: adjust the directory path or deployment permissions so the service user can read the directory. Distributor does not enforce owner, group, or mode policy beyond OS read access.
|
||||
|
||||
## `secret filename ... is invalid`
|
||||
|
||||
Likely cause: a regular file in `secrets.directory` does not match `[A-Za-z_][A-Za-z0-9_]*`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
find <secrets-directory> -maxdepth 1 -type f -printf '%f\n'
|
||||
```
|
||||
|
||||
Safe fix: rename the file to a valid credential environment variable name, or remove it from the secrets directory.
|
||||
|
||||
## `credential environment variable ... is not set`
|
||||
|
||||
Likely cause: a backend credential field references an environment variable that is absent from both the real process environment and the configured secrets directory.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
env | cut -d= -f1 | rg '^<variable-name>$'
|
||||
ls -l <secrets-directory>/<variable-name>
|
||||
```
|
||||
|
||||
Safe fix: set the real environment variable or create a readable secrets-directory file with the same name.
|
||||
|
||||
## `secret ... ignored because the real environment already has that variable`
|
||||
|
||||
Likely cause: the real process environment and secrets directory both define the variable with different values.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
env | cut -d= -f1 | rg '^<variable-name>$'
|
||||
ls -l <secrets-directory>/<variable-name>
|
||||
```
|
||||
|
||||
Safe fix: remove one source of the credential or make the deployment intentionally prefer the real environment value. Distributor does not print either value.
|
||||
|
||||
## `host is required for ssh backend`
|
||||
|
||||
Likely cause: SSH config is missing the structured `host` field, or an old URL-style SSH config is still in use.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: configure SSH with `host`, optional `user` and `port`, and `path`. SSH URLs are not part of the active config schema.
|
||||
|
||||
## `no SSH auth methods configured`
|
||||
|
||||
Likely cause: neither an SSH agent nor `ssh_key_file` is available.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
test -n "$SSH_AUTH_SOCK" && ssh-add -l
|
||||
ls -l <ssh-key-file>
|
||||
```
|
||||
|
||||
Safe fix: start an SSH agent with an appropriate key loaded, or configure `ssh_key_file` with a readable private key.
|
||||
|
||||
## `host key ... is unknown` or `known_hosts is required`
|
||||
|
||||
Likely cause: strict host key checking has no known host key, or `accept-new` cannot persist a new key.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ls -l <known-hosts-path>
|
||||
ssh-keygen -F <host> -f <known-hosts-path>
|
||||
```
|
||||
|
||||
Safe fix: configure a writable `known_hosts` path for `accept-new`, pre-populate `known_hosts` for `strict`, or explicitly use `host_key_policy: off` only for insecure test environments.
|
||||
|
||||
## `host key ... has changed`
|
||||
|
||||
Likely cause: the remote server presented a different host key than the one recorded in `known_hosts`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ssh-keygen -F <host> -f <known-hosts-path>
|
||||
```
|
||||
|
||||
Safe fix: verify the server identity out of band before updating `known_hosts`. Do not switch to `host_key_policy: off` to bypass an unexpected changed key.
|
||||
|
||||
## `stat ssh ... not_found` or `no bundles found`
|
||||
|
||||
Likely cause: the configured SSH `path` is wrong, unreadable, or does not contain source bundles.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
sftp <user>@<host>
|
||||
```
|
||||
|
||||
Safe fix: correct the remote root `path`, permissions, or source bundle location.
|
||||
|
||||
## `validate command requires a path` or `inspect command requires a path`
|
||||
|
||||
@@ -95,7 +257,7 @@ Diagnostic:
|
||||
find <destination-path> -maxdepth 2 -print
|
||||
```
|
||||
|
||||
Safe fix: choose an empty destination path or move existing files aside after confirming they are not needed. There is no force overwrite option.
|
||||
Safe fix: choose an empty destination path or move existing files aside after confirming they are not needed. If the destination should be claimed by distributor, preview with `run --dry-run --force` and publish with `run --force` only after confirming the reported `force_replace` action is bounded to the intended bundle path.
|
||||
|
||||
## `fail_conflict`
|
||||
|
||||
@@ -108,7 +270,31 @@ cat <destination-path>/.distributor.json
|
||||
go run ./cmd/distributor inspect <source-root>
|
||||
```
|
||||
|
||||
Safe fix: verify you are publishing the intended source to the intended destination. Use a separate destination path for unrelated content.
|
||||
Safe fix: verify you are publishing the intended source to the intended destination. Use a separate destination path for unrelated content. If the existing state should be replaced, configure `transfer.on_conflict: replace`, preview with `run --dry-run --force`, then publish with `run --force`.
|
||||
|
||||
## `destination is newer and replacement requires --force`
|
||||
|
||||
Likely cause: config explicitly allows newer-destination replacement, but the current run did not include `--force`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run --force
|
||||
```
|
||||
|
||||
Safe fix: prefer the default `transfer.on_destination_newer: skip` unless replacing newer destination state is intentional. To replace it, keep `transfer.on_destination_newer: replace`, confirm the dry-run output shows `force_replace`, then run with `--force`.
|
||||
|
||||
## `force_replace`
|
||||
|
||||
Likely cause: the current run used `--force` and publish planning selected a supported destructive replacement.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run --force
|
||||
```
|
||||
|
||||
Safe fix: inspect the printed pipeline id, destination id, backend, and bundle path. Proceed only if deleting all content within that destination bundle path is intended.
|
||||
|
||||
## `destination output path collision`
|
||||
|
||||
@@ -124,7 +310,7 @@ Safe fix: adjust the source bundle contents or publish policy so source and gene
|
||||
|
||||
## A run failed after writing some files
|
||||
|
||||
Likely cause: a write failed partway through publication. Local execution attempts to clean up outputs written during the failed attempt.
|
||||
Likely cause: a write failed partway through publication. Local, SSH, and S3 execution attempt to clean up outputs written during the failed attempt.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -132,4 +318,4 @@ Diagnostic:
|
||||
find <destination-path> -maxdepth 2 -print
|
||||
```
|
||||
|
||||
Safe fix: inspect the destination before retrying. If only unrelated unmanaged files remain, move them aside or choose a clean destination. Re-run with `--dry-run` before publishing again. See [operations](operations.md).
|
||||
Safe fix: use the pipeline id, destination id, backend, and bundle path printed in the run error to inspect the destination before retrying. If only unrelated unmanaged files remain, move them aside or choose a clean destination. Re-run with `--dry-run` before publishing again. See [operations](operations.md).
|
||||
|
||||
22
examples/s3-destination.yml
Normal file
22
examples/s3-destination.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
# Environment-gated example.
|
||||
# Replace endpoint, bucket, prefix, and credential environment variable names
|
||||
# with values for an S3-compatible service you control before running this config.
|
||||
pipelines:
|
||||
- id: example-s3-destination
|
||||
source:
|
||||
backend: local
|
||||
path: examples/source-bundle
|
||||
destinations:
|
||||
- id: s3-archive
|
||||
backend: s3
|
||||
endpoint: https://s3.example.com
|
||||
bucket: reports
|
||||
prefix: distributor/archive
|
||||
region: us-east-1
|
||||
force_path_style: true
|
||||
credentials:
|
||||
access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
|
||||
secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
21
examples/ssh-destination.yml
Normal file
21
examples/ssh-destination.yml
Normal file
@@ -0,0 +1,21 @@
|
||||
# Environment-gated example.
|
||||
# Replace host, user, path, ssh_key_file, and known_hosts with values for an
|
||||
# SSH/SFTP endpoint you control before running this config.
|
||||
pipelines:
|
||||
- id: example-ssh-destination
|
||||
source:
|
||||
backend: local
|
||||
path: examples/source-bundle
|
||||
destinations:
|
||||
- id: ssh-archive
|
||||
backend: ssh
|
||||
host: ssh.example.com
|
||||
user: distributor
|
||||
port: 22
|
||||
path: /srv/distributor/archive
|
||||
ssh_key_file: /home/distributor/.ssh/id_ed25519
|
||||
known_hosts: /home/distributor/.ssh/known_hosts
|
||||
host_key_policy: strict
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
25
go.mod
25
go.mod
@@ -3,6 +3,31 @@ module gitea.maximumdirect.net/eric/distributor
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.9
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.20
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.19
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2
|
||||
github.com/aws/smithy-go v1.26.0
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/yuin/goldmark v1.8.2
|
||||
golang.org/x/crypto v0.52.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
)
|
||||
|
||||
52
go.sum
52
go.sum
@@ -1,5 +1,57 @@
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.20 h1:8VMDnWc/kEzxsI/1ngGM9mG81a8IGmIHD8KLcYGwagc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.20/go.mod h1:PuwEpciweIXGULWeOeSTXtSbH4CW9mWdWrhdCKQI1sM=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.19 h1:yuFzSV1U0aRNYCQGVaTY2zW2M/L93pYHnXnrJUphYhU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.19/go.mod h1:7y63L1kGzeoDlJaQ3Z578KrnmfBut96JjvJUzGwR+YE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 h1:0w6dCiO8iez+YKwRhRBlL1CH/E3GTfdkuzrwj1by8vo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25/go.mod h1:9FDWUothyr5RCRAHc45XOiVCzUR8n/IhCYX+uVqw6vk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 h1:Uii3frf9ztec/ABM2/FSH9/z7PLzxfpG8h4RpkUFflQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25/go.mod h1:G6kntsA2GorAxDPbap6xgB2F+amSLUF8GJTi7PUoX44=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 h1:r1+/l6m+WaUJF9HISEsNOLHSNj5EXYQxK8VX6Cz9NlA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25/go.mod h1:cKf+D+NMDK1LndD7BowHbBZPgR9V0/5HubH0PFWvA+c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 h1:A1PmWU2zfkIm9EyFlJncFXL4W4phML+h8KjltUsCvNQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26/go.mod h1:dY4MRzXEizrD4hqtpKvWVGPX7QleSGGVY+EBolo1RmM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 h1:d5/908OJ4bXg8lyjeMPvXetEKqoDoLi5Owy1zNue3yg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 h1:dD3dhHNglpd98gs72my22Ndqi1hqQGllFFg1F+twfxg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25/go.mod h1:0yAbjPfd64gG7mj85RW+fMEYdfBgCRZw8g/oWcL1pjc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 h1:1VwbP3qMNfxUDEXWki4rCE5iA+44VA1lokTz9HasGzw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.1.1/go.mod h1:vUtyoSj0OPji3kjIVSc/GlKuWEiL33f/WFxl6dmpy/A=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 h1:N6pIsdFOW1Kd9S4KyFKXdGRBojPPxkP32+uHFWLv4Hc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.19/go.mod h1:3gt5WJArFooNmyLONS+h/R4J+o86II8du38IgCwj9dE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 h1:hc+lBYiiTr8Zk4MTzIsQ92MeDWCIDvWGmzKUWOaBcOg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2/go.mod h1:hU6fqB3OJA6/ePheD47LQnxvjYk6br6PtQxs+Q9ojvk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 h1:ErklX/7uhSbkAAeyQD/Y1OoQ9hO3SJXQNEgksORW3Js=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.3/go.mod h1:ULe4HCzfKPiR6R3HEurE3b1upEkuk8AkMrOKtaOxKO8=
|
||||
github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s=
|
||||
github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
|
||||
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
|
||||
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -267,6 +267,51 @@ func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, ma
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) DeletePrefix(ctx context.Context, prefix string, opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
nativePrefix, err := b.nativePath(prefix, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := b.rejectSymlinkAncestors(nativePrefix, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if prefix == "" {
|
||||
entries, err := os.ReadDir(nativePrefix)
|
||||
if err != nil {
|
||||
if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return b.translateError(storage.OpDeletePrefix, prefix, err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
child := filepath.Join(nativePrefix, entry.Name())
|
||||
if err := os.RemoveAll(child); err != nil {
|
||||
return b.translateError(storage.OpDeletePrefix, entry.Name(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Lstat(nativePrefix); err != nil {
|
||||
if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return b.translateError(storage.OpDeletePrefix, prefix, err)
|
||||
}
|
||||
if err := os.RemoveAll(nativePrefix); err != nil {
|
||||
return b.translateError(storage.OpDeletePrefix, prefix, err)
|
||||
}
|
||||
if opts.PruneEmptyDirs {
|
||||
b.pruneEmptyParents(filepath.Dir(nativePrefix))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) {
|
||||
if logicalPath == "" {
|
||||
if !allowEmpty {
|
||||
|
||||
@@ -169,6 +169,30 @@ func TestBackendManagedDeletion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendDeletePrefixStaysWithinPrefix(t *testing.T) {
|
||||
backend := newBackend(t)
|
||||
mustWrite(t, backend, "bundle/report.md", "report")
|
||||
mustWrite(t, backend, "bundle/nested/old.txt", "old")
|
||||
mustWrite(t, backend, "bundle-sibling/keep.txt", "keep")
|
||||
mustWrite(t, backend, "outside.txt", "outside")
|
||||
|
||||
if err := backend.DeletePrefix(context.Background(), "bundle", storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
t.Fatalf("DeletePrefix() error = %v", err)
|
||||
}
|
||||
if _, err := backend.Stat(context.Background(), "bundle/report.md"); !storage.IsNotFound(err) {
|
||||
t.Fatalf("deleted file stat error = %v, want not found", err)
|
||||
}
|
||||
if _, err := backend.Stat(context.Background(), "bundle/nested/old.txt"); !storage.IsNotFound(err) {
|
||||
t.Fatalf("deleted nested file stat error = %v, want not found", err)
|
||||
}
|
||||
if _, err := backend.Stat(context.Background(), "bundle-sibling/keep.txt"); err != nil {
|
||||
t.Fatalf("sibling stat error = %v", err)
|
||||
}
|
||||
if _, err := backend.Stat(context.Background(), "outside.txt"); err != nil {
|
||||
t.Fatalf("outside stat error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendHasAny(t *testing.T) {
|
||||
backend := newBackend(t)
|
||||
found, err := backend.HasAny(context.Background(), "missing")
|
||||
|
||||
485
internal/adapters/s3/backend.go
Normal file
485
internal/adapters/s3/backend.go
Normal file
@@ -0,0 +1,485 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awscfg "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
)
|
||||
|
||||
type Client interface {
|
||||
HeadObject(ctx context.Context, input *awss3.HeadObjectInput, optFns ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error)
|
||||
GetObject(ctx context.Context, input *awss3.GetObjectInput, optFns ...func(*awss3.Options)) (*awss3.GetObjectOutput, error)
|
||||
PutObject(ctx context.Context, input *awss3.PutObjectInput, optFns ...func(*awss3.Options)) (*awss3.PutObjectOutput, error)
|
||||
ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, optFns ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error)
|
||||
DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, optFns ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error)
|
||||
}
|
||||
|
||||
type Backend struct {
|
||||
client Client
|
||||
bucket string
|
||||
prefix string
|
||||
}
|
||||
|
||||
func New(ctx context.Context, options Options) (*Backend, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options, err := options.normalized()
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Prefix, storage.ErrInvalidPath, err)
|
||||
}
|
||||
loadOptions := []func(*awscfg.LoadOptions) error{
|
||||
awscfg.WithRegion(options.Region),
|
||||
}
|
||||
if options.AccessKeyID != "" {
|
||||
loadOptions = append(loadOptions, awscfg.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(options.AccessKeyID, options.SecretAccessKey, "")))
|
||||
}
|
||||
cfg, err := awscfg.LoadDefaultConfig(ctx, loadOptions...)
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Endpoint, storage.ErrUnknown, err)
|
||||
}
|
||||
client := awss3.NewFromConfig(cfg, func(o *awss3.Options) {
|
||||
o.BaseEndpoint = aws.String(options.Endpoint)
|
||||
o.UsePathStyle = options.ForcePathStyle
|
||||
})
|
||||
return NewWithClient(client, options)
|
||||
}
|
||||
|
||||
func NewWithClient(client Client, options Options) (*Backend, error) {
|
||||
options, err := options.normalized()
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Prefix, storage.ErrInvalidPath, err)
|
||||
}
|
||||
if client == nil {
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Bucket, storage.ErrInvalidPath, fmt.Errorf("client is required"))
|
||||
}
|
||||
return &Backend{
|
||||
client: client,
|
||||
bucket: options.Bucket,
|
||||
prefix: options.Prefix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *Backend) ReadFile(ctx context.Context, logicalPath string) ([]byte, error) {
|
||||
reader, err := b.OpenReader(ctx, logicalPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpReadFile, BackendName, logicalPath, storage.ErrUnknown, err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (b *Backend) OpenReader(ctx context.Context, logicalPath string) (io.ReadCloser, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := b.objectKey(logicalPath, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
output, err := b.client.GetObject(ctx, &awss3.GetObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
|
||||
}
|
||||
return output.Body, nil
|
||||
}
|
||||
|
||||
func (b *Backend) WriteFile(ctx context.Context, logicalPath string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
|
||||
opts.Size = int64(len(data))
|
||||
opts.SizeKnown = true
|
||||
return b.WriteFrom(ctx, logicalPath, bytes.NewReader(data), opts)
|
||||
}
|
||||
|
||||
func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
key, err := b.objectKey(logicalPath, false)
|
||||
if err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
if !opts.Overwrite {
|
||||
_, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err == nil {
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrAlreadyExist, nil)
|
||||
}
|
||||
if !isNotFound(err) {
|
||||
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
|
||||
}
|
||||
}
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, err)
|
||||
}
|
||||
if opts.SizeKnown && int64(len(data)) != opts.Size {
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", len(data), opts.Size))
|
||||
}
|
||||
contentType := opts.ContentType
|
||||
if contentType == "" {
|
||||
contentType = ContentType(logicalPath)
|
||||
}
|
||||
_, err = b.client.PutObject(ctx, &awss3.PutObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader(data),
|
||||
ContentLength: aws.Int64(int64(len(data))),
|
||||
ContentType: aws.String(contentType),
|
||||
})
|
||||
if err != nil {
|
||||
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
|
||||
}
|
||||
return storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: int64(len(data))}, nil
|
||||
}
|
||||
|
||||
func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
key, err := b.objectKey(logicalPath, false)
|
||||
if err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
output, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return storage.Entry{}, b.translateError(storage.OpStat, logicalPath, err)
|
||||
}
|
||||
size := int64(0)
|
||||
if output.ContentLength != nil {
|
||||
size = *output.ContentLength
|
||||
}
|
||||
return storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: size}, nil
|
||||
}
|
||||
|
||||
func (b *Backend) Walk(ctx context.Context, logicalPrefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := storage.ValidatePrefix(logicalPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
visited := 0
|
||||
emit := func(entry storage.Entry) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Limit > 0 && visited >= opts.Limit {
|
||||
return storage.ErrStopWalk
|
||||
}
|
||||
visited++
|
||||
if err := fn(entry); err != nil {
|
||||
if errors.Is(err, storage.ErrStopWalk) {
|
||||
return storage.ErrStopWalk
|
||||
}
|
||||
return storage.NewError(storage.OpWalk, BackendName, entry.Path, storage.ErrUnknown, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if logicalPrefix != "" {
|
||||
entry, err := b.Stat(ctx, logicalPrefix)
|
||||
if err == nil {
|
||||
if err := emit(entry); errors.Is(err, storage.ErrStopWalk) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Limit > 0 && visited >= opts.Limit {
|
||||
return nil
|
||||
}
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err := b.walkObjects(ctx, logicalPrefix, opts, emit)
|
||||
if errors.Is(err, storage.ErrStopWalk) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *Backend) HasAny(ctx context.Context, logicalPrefix string) (bool, error) {
|
||||
found := false
|
||||
err := b.Walk(ctx, logicalPrefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
|
||||
found = true
|
||||
return storage.ErrStopWalk
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, target := range targets {
|
||||
if err := b.deleteObject(ctx, storage.OpDeleteManagedBundle, target, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) DeletePrefix(ctx context.Context, logicalPrefix string, opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := storage.ValidatePrefix(logicalPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
found := false
|
||||
if logicalPrefix != "" {
|
||||
key, err := b.objectKey(logicalPrefix, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = b.client.HeadObject(ctx, &awss3.HeadObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err == nil {
|
||||
found = true
|
||||
if err := b.deleteObject(ctx, storage.OpDeletePrefix, logicalPrefix, storage.DeleteOptions{IgnoreMissing: false}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !isNotFound(err) {
|
||||
return b.translateError(storage.OpDeletePrefix, logicalPrefix, err)
|
||||
}
|
||||
}
|
||||
var entries []storage.Entry
|
||||
if err := b.walkObjects(ctx, logicalPrefix, storage.WalkOptions{Recursive: true}, func(entry storage.Entry) error {
|
||||
if entry.Type == storage.EntryTypeFile {
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
found = true
|
||||
if err := b.deleteObject(ctx, storage.OpDeletePrefix, entry.Path, storage.DeleteOptions{IgnoreMissing: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !found && !opts.IgnoreMissing {
|
||||
return storage.NewError(storage.OpDeletePrefix, BackendName, logicalPrefix, storage.ErrNotFound, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) deleteObject(ctx context.Context, op, logicalPath string, opts storage.DeleteOptions) error {
|
||||
key, err := b.objectKey(logicalPath, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if key == b.prefix {
|
||||
return storage.NewError(op, BackendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||
}
|
||||
if !opts.IgnoreMissing {
|
||||
_, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return b.translateError(op, logicalPath, err)
|
||||
}
|
||||
}
|
||||
_, err = b.client.DeleteObject(ctx, &awss3.DeleteObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
if opts.IgnoreMissing && isNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return b.translateError(op, logicalPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) walkObjects(ctx context.Context, logicalPrefix string, opts storage.WalkOptions, emit func(storage.Entry) error) error {
|
||||
listPrefix, err := b.listPrefix(logicalPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delimiter := ""
|
||||
if !opts.Recursive {
|
||||
delimiter = "/"
|
||||
}
|
||||
var token *string
|
||||
for {
|
||||
output, err := b.client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Prefix: aws.String(listPrefix),
|
||||
Delimiter: aws.String(delimiter),
|
||||
ContinuationToken: token,
|
||||
})
|
||||
if err != nil {
|
||||
return b.translateError(storage.OpWalk, logicalPrefix, err)
|
||||
}
|
||||
entries := entriesFromList(logicalPrefix, b.prefix, output)
|
||||
sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path })
|
||||
for _, entry := range entries {
|
||||
if entry.Path == "" {
|
||||
continue
|
||||
}
|
||||
if err := emit(entry); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if output.IsTruncated == nil || !*output.IsTruncated {
|
||||
return nil
|
||||
}
|
||||
token = output.NextContinuationToken
|
||||
}
|
||||
}
|
||||
|
||||
func entriesFromList(logicalPrefix, rootPrefix string, output *awss3.ListObjectsV2Output) []storage.Entry {
|
||||
seen := make(map[string]storage.Entry)
|
||||
for _, object := range output.Contents {
|
||||
if object.Key == nil {
|
||||
continue
|
||||
}
|
||||
logicalPath := logicalPathFromKey(rootPrefix, *object.Key)
|
||||
if logicalPath == "" || logicalPath == logicalPrefix {
|
||||
continue
|
||||
}
|
||||
size := int64(0)
|
||||
if object.Size != nil {
|
||||
size = *object.Size
|
||||
}
|
||||
seen[logicalPath] = storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: size}
|
||||
}
|
||||
for _, commonPrefix := range output.CommonPrefixes {
|
||||
if commonPrefix.Prefix == nil {
|
||||
continue
|
||||
}
|
||||
logicalPath := strings.TrimSuffix(logicalPathFromKey(rootPrefix, *commonPrefix.Prefix), "/")
|
||||
if logicalPath == "" || logicalPath == logicalPrefix {
|
||||
continue
|
||||
}
|
||||
seen[logicalPath] = storage.Entry{Path: logicalPath, Type: storage.EntryTypeDirectory}
|
||||
}
|
||||
entries := make([]storage.Entry, 0, len(seen))
|
||||
for _, entry := range seen {
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func (b *Backend) objectKey(logicalPath string, allowEmpty bool) (string, error) {
|
||||
if logicalPath == "" {
|
||||
if !allowEmpty {
|
||||
return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||
}
|
||||
return b.prefix, nil
|
||||
}
|
||||
if err := storage.ValidatePath(logicalPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if b.prefix == "" {
|
||||
return logicalPath, nil
|
||||
}
|
||||
return b.prefix + "/" + logicalPath, nil
|
||||
}
|
||||
|
||||
func (b *Backend) listPrefix(logicalPrefix string) (string, error) {
|
||||
key, err := b.objectKey(logicalPrefix, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if key != "" {
|
||||
key = strings.TrimSuffix(key, "/") + "/"
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func logicalPathFromKey(rootPrefix, key string) string {
|
||||
if rootPrefix == "" {
|
||||
return key
|
||||
}
|
||||
if key == rootPrefix {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimPrefix(key, rootPrefix+"/")
|
||||
}
|
||||
|
||||
func ContentType(logicalPath string) string {
|
||||
switch strings.ToLower(path.Ext(logicalPath)) {
|
||||
case ".md":
|
||||
return "text/markdown; charset=utf-8"
|
||||
case ".html":
|
||||
return "text/html; charset=utf-8"
|
||||
case ".json":
|
||||
return "application/json"
|
||||
case ".txt":
|
||||
return "text/plain; charset=utf-8"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var notFound *types.NotFound
|
||||
if errors.As(err, ¬Found) {
|
||||
return true
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
switch apiErr.ErrorCode() {
|
||||
case "NotFound", "NoSuchKey", "404":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (b *Backend) translateError(op, logicalPath string, err error) error {
|
||||
kind := storage.ErrUnknown
|
||||
if isNotFound(err) {
|
||||
kind = storage.ErrNotFound
|
||||
} else {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
switch apiErr.ErrorCode() {
|
||||
case "AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch":
|
||||
kind = storage.ErrPermission
|
||||
case "SlowDown", "RequestTimeout", "ServiceUnavailable", "InternalError":
|
||||
kind = storage.ErrTemporary
|
||||
case "InvalidBucketName", "NoSuchBucket":
|
||||
kind = storage.ErrInvalidPath
|
||||
}
|
||||
}
|
||||
}
|
||||
return storage.NewError(op, BackendName, logicalPath, kind, err)
|
||||
}
|
||||
442
internal/adapters/s3/backend_test.go
Normal file
442
internal/adapters/s3/backend_test.go
Normal file
@@ -0,0 +1,442 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
func TestKeyAndPrefixNormalization(t *testing.T) {
|
||||
backend := newTestBackend(t, "root/prefix", nil)
|
||||
key, err := backend.objectKey("bundle/report.md", false)
|
||||
if err != nil {
|
||||
t.Fatalf("objectKey() error = %v", err)
|
||||
}
|
||||
if got, want := key, "root/prefix/bundle/report.md"; got != want {
|
||||
t.Fatalf("objectKey() = %q, want %q", got, want)
|
||||
}
|
||||
listPrefix, err := backend.listPrefix("bundle")
|
||||
if err != nil {
|
||||
t.Fatalf("listPrefix() error = %v", err)
|
||||
}
|
||||
if got, want := listPrefix, "root/prefix/bundle/"; got != want {
|
||||
t.Fatalf("listPrefix() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathTraversalRejected(t *testing.T) {
|
||||
backend := newTestBackend(t, "", nil)
|
||||
for _, logicalPath := range []string{"/absolute", "../escape", "a/../b", `a\b`} {
|
||||
t.Run(logicalPath, func(t *testing.T) {
|
||||
if _, err := backend.objectKey(logicalPath, false); err == nil || !storage.IsInvalidPath(err) {
|
||||
t.Fatalf("objectKey() error = %v, want invalid path", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"report.md": "text/markdown; charset=utf-8",
|
||||
"report.html": "text/html; charset=utf-8",
|
||||
"state.json": "application/json",
|
||||
"summary.txt": "text/plain; charset=utf-8",
|
||||
"data.bin": "application/octet-stream",
|
||||
}
|
||||
for path, want := range tests {
|
||||
if got := ContentType(path); got != want {
|
||||
t.Fatalf("ContentType(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatRequiresExactObject(t *testing.T) {
|
||||
client := newFakeClient(map[string]string{"root/dir/file.txt": "data"})
|
||||
backend := newTestBackend(t, "root", client)
|
||||
_, err := backend.Stat(context.Background(), "dir")
|
||||
if err == nil || !storage.IsNotFound(err) {
|
||||
t.Fatalf("Stat() error = %v, want not found", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkIncludesExactObjectAndDescendants(t *testing.T) {
|
||||
client := newFakeClient(map[string]string{
|
||||
"root/bundle": "marker",
|
||||
"root/bundle/report.md": "report",
|
||||
})
|
||||
backend := newTestBackend(t, "root", client)
|
||||
entries, err := storage.List(context.Background(), backend, "bundle", storage.WalkOptions{Recursive: true})
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
paths := entryPaths(entries)
|
||||
if got, want := paths, []string{"bundle", "bundle/report.md"}; !equalStrings(got, want) {
|
||||
t.Fatalf("paths = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkExactObjectHonorsLimitBeforeListingDescendants(t *testing.T) {
|
||||
client := newFakeClient(map[string]string{
|
||||
"root/bundle": "marker",
|
||||
"root/bundle/report.md": "report",
|
||||
})
|
||||
backend := newTestBackend(t, "root", client)
|
||||
var entries []storage.Entry
|
||||
err := backend.Walk(context.Background(), "bundle", storage.WalkOptions{Recursive: true, Limit: 1}, func(entry storage.Entry) error {
|
||||
entries = append(entries, entry)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Walk() error = %v", err)
|
||||
}
|
||||
paths := entryPaths(entries)
|
||||
if got, want := paths, []string{"bundle"}; !equalStrings(got, want) {
|
||||
t.Fatalf("paths = %v, want %v", got, want)
|
||||
}
|
||||
if len(client.tokens) != 0 {
|
||||
t.Fatalf("list calls = %d, want none", len(client.tokens))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasAnyWithExactObjectStopsBeforeListingDescendants(t *testing.T) {
|
||||
client := newFakeClient(map[string]string{
|
||||
"root/bundle": "marker",
|
||||
"root/bundle/report.md": "report",
|
||||
})
|
||||
backend := newTestBackend(t, "root", client)
|
||||
found, err := backend.HasAny(context.Background(), "bundle")
|
||||
if err != nil {
|
||||
t.Fatalf("HasAny() error = %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("HasAny() = false, want true")
|
||||
}
|
||||
if len(client.tokens) != 0 {
|
||||
t.Fatalf("list calls = %d, want none", len(client.tokens))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFromChecksOverwriteBeforePut(t *testing.T) {
|
||||
client := newFakeClient(map[string]string{"root/report.md": "old"})
|
||||
backend := newTestBackend(t, "root", client)
|
||||
_, err := backend.WriteFile(context.Background(), "report.md", []byte("new"), storage.WriteOptions{})
|
||||
if err == nil || !storage.IsAlreadyExists(err) {
|
||||
t.Fatalf("WriteFile() error = %v, want already exists", err)
|
||||
}
|
||||
if len(client.putKeys) != 0 {
|
||||
t.Fatalf("put keys = %v, want none", client.putKeys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFromPutsNewObjectWithContentType(t *testing.T) {
|
||||
client := newFakeClient(nil)
|
||||
backend := newTestBackend(t, "root", client)
|
||||
entry, err := backend.WriteFile(context.Background(), "report.html", []byte("<p>ok</p>"), storage.WriteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
if entry.Path != "report.html" || entry.Size != 9 || entry.Type != storage.EntryTypeFile {
|
||||
t.Fatalf("entry = %#v", entry)
|
||||
}
|
||||
if got, want := client.objects["root/report.html"], "<p>ok</p>"; got != want {
|
||||
t.Fatalf("object = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := client.contentTypes["root/report.html"], "text/html; charset=utf-8"; got != want {
|
||||
t.Fatalf("content type = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkUsesPagination(t *testing.T) {
|
||||
client := newFakeClient(nil)
|
||||
client.listPages = []awss3.ListObjectsV2Output{
|
||||
{
|
||||
Contents: []types.Object{{Key: aws.String("root/a.txt"), Size: aws.Int64(1)}},
|
||||
IsTruncated: aws.Bool(true),
|
||||
NextContinuationToken: aws.String("next"),
|
||||
},
|
||||
{
|
||||
Contents: []types.Object{{Key: aws.String("root/b.txt"), Size: aws.Int64(2)}},
|
||||
IsTruncated: aws.Bool(false),
|
||||
},
|
||||
}
|
||||
backend := newTestBackend(t, "root", client)
|
||||
entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: true})
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
paths := entryPaths(entries)
|
||||
if got, want := paths, []string{"a.txt", "b.txt"}; !equalStrings(got, want) {
|
||||
t.Fatalf("paths = %v, want %v", got, want)
|
||||
}
|
||||
if got, want := client.tokens, []string{"", "next"}; !equalStrings(got, want) {
|
||||
t.Fatalf("tokens = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasAnyStopsAfterFirstPage(t *testing.T) {
|
||||
client := newFakeClient(nil)
|
||||
client.listPages = []awss3.ListObjectsV2Output{
|
||||
{
|
||||
Contents: []types.Object{{Key: aws.String("root/a.txt"), Size: aws.Int64(1)}},
|
||||
IsTruncated: aws.Bool(true),
|
||||
NextContinuationToken: aws.String("next"),
|
||||
},
|
||||
{
|
||||
Contents: []types.Object{{Key: aws.String("root/b.txt"), Size: aws.Int64(2)}},
|
||||
IsTruncated: aws.Bool(false),
|
||||
},
|
||||
}
|
||||
backend := newTestBackend(t, "root", client)
|
||||
found, err := backend.HasAny(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("HasAny() error = %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("HasAny() = false, want true")
|
||||
}
|
||||
if got, want := len(client.tokens), 1; got != want {
|
||||
t.Fatalf("list calls = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkNonRecursiveUsesPrefixBoundary(t *testing.T) {
|
||||
client := newFakeClient(map[string]string{
|
||||
"base/dir/file.txt": "nested",
|
||||
"base/file.txt": "file",
|
||||
"baseball/file.txt": "wrong",
|
||||
})
|
||||
backend := newTestBackend(t, "base", client)
|
||||
entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
paths := entryPaths(entries)
|
||||
if got, want := paths, []string{"dir", "file.txt"}; !equalStrings(got, want) {
|
||||
t.Fatalf("paths = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteManagedBundleDeletesOnlyManagedTargets(t *testing.T) {
|
||||
client := newFakeClient(map[string]string{
|
||||
"root/report.md": "report",
|
||||
"root/.distributor.json": "state",
|
||||
"root/keep.txt": "keep",
|
||||
})
|
||||
backend := newTestBackend(t, "root", client)
|
||||
err := backend.DeleteManagedBundle(context.Background(), "", []string{"report.md"}, storage.DeleteOptions{IgnoreMissing: true})
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteManagedBundle() error = %v", err)
|
||||
}
|
||||
if _, ok := client.objects["root/report.md"]; ok {
|
||||
t.Fatal("managed output still exists")
|
||||
}
|
||||
if _, ok := client.objects["root/.distributor.json"]; ok {
|
||||
t.Fatal("state file still exists")
|
||||
}
|
||||
if _, ok := client.objects["root/keep.txt"]; !ok {
|
||||
t.Fatal("unmanaged object was deleted")
|
||||
}
|
||||
if got, want := sortedStrings(client.deleteKeys), []string{"root/.distributor.json", "root/report.md"}; !equalStrings(got, want) {
|
||||
t.Fatalf("deleted keys = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletePrefixStaysWithinPrefix(t *testing.T) {
|
||||
client := newFakeClient(map[string]string{
|
||||
"root/bundle/report.md": "report",
|
||||
"root/bundle/nested/old.txt": "old",
|
||||
"root/bundle-sibling/keep.txt": "keep",
|
||||
"root/outside.txt": "outside",
|
||||
"other-root/bundle/report.md": "other",
|
||||
"root/.distributor-prefix-marker": "marker",
|
||||
})
|
||||
backend := newTestBackend(t, "root", client)
|
||||
if err := backend.DeletePrefix(context.Background(), "bundle", storage.DeleteOptions{IgnoreMissing: true}); err != nil {
|
||||
t.Fatalf("DeletePrefix() error = %v", err)
|
||||
}
|
||||
for _, deleted := range []string{"root/bundle/report.md", "root/bundle/nested/old.txt"} {
|
||||
if _, ok := client.objects[deleted]; ok {
|
||||
t.Fatalf("%s still exists", deleted)
|
||||
}
|
||||
}
|
||||
for _, kept := range []string{"root/bundle-sibling/keep.txt", "root/outside.txt", "other-root/bundle/report.md", "root/.distributor-prefix-marker"} {
|
||||
if _, ok := client.objects[kept]; !ok {
|
||||
t.Fatalf("%s was deleted", kept)
|
||||
}
|
||||
}
|
||||
if got, want := sortedStrings(client.deleteKeys), []string{"root/bundle/nested/old.txt", "root/bundle/report.md"}; !equalStrings(got, want) {
|
||||
t.Fatalf("deleted keys = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestBackend(t *testing.T, prefix string, client *fakeClient) *Backend {
|
||||
t.Helper()
|
||||
if client == nil {
|
||||
client = newFakeClient(nil)
|
||||
}
|
||||
backend, err := NewWithClient(client, Options{
|
||||
Endpoint: "https://s3.example.com",
|
||||
Bucket: "reports",
|
||||
Prefix: prefix,
|
||||
Region: DefaultRegion,
|
||||
ForcePathStyle: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewWithClient() error = %v", err)
|
||||
}
|
||||
return backend
|
||||
}
|
||||
|
||||
type fakeClient struct {
|
||||
objects map[string]string
|
||||
contentTypes map[string]string
|
||||
listPages []awss3.ListObjectsV2Output
|
||||
tokens []string
|
||||
putKeys []string
|
||||
deleteKeys []string
|
||||
}
|
||||
|
||||
func newFakeClient(objects map[string]string) *fakeClient {
|
||||
copied := make(map[string]string)
|
||||
for key, value := range objects {
|
||||
copied[key] = value
|
||||
}
|
||||
return &fakeClient{
|
||||
objects: copied,
|
||||
contentTypes: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *fakeClient) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, optFns ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, ok := c.objects[aws.ToString(input.Key)]
|
||||
if !ok {
|
||||
return nil, &types.NotFound{}
|
||||
}
|
||||
return &awss3.HeadObjectOutput{ContentLength: aws.Int64(int64(len(value)))}, nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) GetObject(ctx context.Context, input *awss3.GetObjectInput, optFns ...func(*awss3.Options)) (*awss3.GetObjectOutput, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, ok := c.objects[aws.ToString(input.Key)]
|
||||
if !ok {
|
||||
return nil, &types.NotFound{}
|
||||
}
|
||||
return &awss3.GetObjectOutput{
|
||||
Body: io.NopCloser(stringsReader(value)),
|
||||
ContentLength: aws.Int64(int64(len(value))),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) PutObject(ctx context.Context, input *awss3.PutObjectInput, optFns ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := io.ReadAll(input.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := aws.ToString(input.Key)
|
||||
c.objects[key] = string(data)
|
||||
c.contentTypes[key] = aws.ToString(input.ContentType)
|
||||
c.putKeys = append(c.putKeys, key)
|
||||
return &awss3.PutObjectOutput{}, nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, optFns ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.tokens = append(c.tokens, aws.ToString(input.ContinuationToken))
|
||||
if len(c.listPages) > 0 {
|
||||
index := len(c.tokens) - 1
|
||||
if index >= len(c.listPages) {
|
||||
return &awss3.ListObjectsV2Output{IsTruncated: aws.Bool(false)}, nil
|
||||
}
|
||||
page := c.listPages[index]
|
||||
return &page, nil
|
||||
}
|
||||
return c.dynamicList(input), nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, optFns ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := aws.ToString(input.Key)
|
||||
delete(c.objects, key)
|
||||
c.deleteKeys = append(c.deleteKeys, key)
|
||||
return &awss3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) dynamicList(input *awss3.ListObjectsV2Input) *awss3.ListObjectsV2Output {
|
||||
prefix := aws.ToString(input.Prefix)
|
||||
delimiter := aws.ToString(input.Delimiter)
|
||||
var contents []types.Object
|
||||
commonPrefixes := make(map[string]struct{})
|
||||
for key, value := range c.objects {
|
||||
if !strings.HasPrefix(key, prefix) {
|
||||
continue
|
||||
}
|
||||
remainder := strings.TrimPrefix(key, prefix)
|
||||
if delimiter != "" {
|
||||
if index := strings.Index(remainder, delimiter); index >= 0 {
|
||||
commonPrefixes[prefix+remainder[:index+1]] = struct{}{}
|
||||
continue
|
||||
}
|
||||
}
|
||||
contents = append(contents, types.Object{Key: aws.String(key), Size: aws.Int64(int64(len(value)))})
|
||||
}
|
||||
sort.Slice(contents, func(i, j int) bool { return aws.ToString(contents[i].Key) < aws.ToString(contents[j].Key) })
|
||||
prefixes := make([]types.CommonPrefix, 0, len(commonPrefixes))
|
||||
for prefix := range commonPrefixes {
|
||||
prefixes = append(prefixes, types.CommonPrefix{Prefix: aws.String(prefix)})
|
||||
}
|
||||
sort.Slice(prefixes, func(i, j int) bool { return aws.ToString(prefixes[i].Prefix) < aws.ToString(prefixes[j].Prefix) })
|
||||
return &awss3.ListObjectsV2Output{
|
||||
Contents: contents,
|
||||
CommonPrefixes: prefixes,
|
||||
IsTruncated: aws.Bool(false),
|
||||
}
|
||||
}
|
||||
|
||||
func stringsReader(value string) io.Reader {
|
||||
return strings.NewReader(value)
|
||||
}
|
||||
|
||||
func entryPaths(entries []storage.Entry) []string {
|
||||
paths := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
paths = append(paths, entry.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func sortedStrings(values []string) []string {
|
||||
copied := append([]string(nil), values...)
|
||||
sort.Strings(copied)
|
||||
return copied
|
||||
}
|
||||
|
||||
func equalStrings(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for index := range a {
|
||||
if a[index] != b[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
39
internal/adapters/s3/integration_test.go
Normal file
39
internal/adapters/s3/integration_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIntegrationS3BackendHasAny(t *testing.T) {
|
||||
endpoint := os.Getenv("DISTRIBUTOR_TEST_S3_ENDPOINT")
|
||||
bucket := os.Getenv("DISTRIBUTOR_TEST_S3_BUCKET")
|
||||
if endpoint == "" || bucket == "" {
|
||||
t.Skip("DISTRIBUTOR_TEST_S3_ENDPOINT and DISTRIBUTOR_TEST_S3_BUCKET are not set")
|
||||
}
|
||||
forcePathStyle := true
|
||||
if raw := os.Getenv("DISTRIBUTOR_TEST_S3_FORCE_PATH_STYLE"); raw != "" {
|
||||
parsed, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse DISTRIBUTOR_TEST_S3_FORCE_PATH_STYLE: %v", err)
|
||||
}
|
||||
forcePathStyle = parsed
|
||||
}
|
||||
backend, err := New(context.Background(), Options{
|
||||
Endpoint: endpoint,
|
||||
Bucket: bucket,
|
||||
Prefix: os.Getenv("DISTRIBUTOR_TEST_S3_PREFIX"),
|
||||
Region: os.Getenv("DISTRIBUTOR_TEST_S3_REGION"),
|
||||
ForcePathStyle: forcePathStyle,
|
||||
AccessKeyID: os.Getenv("DISTRIBUTOR_TEST_S3_ACCESS_KEY_ID"),
|
||||
SecretAccessKey: os.Getenv("DISTRIBUTOR_TEST_S3_SECRET_ACCESS_KEY"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
if _, err := backend.HasAny(context.Background(), ""); err != nil {
|
||||
t.Fatalf("HasAny(root) error = %v", err)
|
||||
}
|
||||
}
|
||||
42
internal/adapters/s3/options.go
Normal file
42
internal/adapters/s3/options.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
const BackendName = "s3"
|
||||
|
||||
const DefaultRegion = "us-east-1"
|
||||
|
||||
type Options struct {
|
||||
Endpoint string
|
||||
Bucket string
|
||||
Prefix string
|
||||
Region string
|
||||
ForcePathStyle bool
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
}
|
||||
|
||||
func (o Options) normalized() (Options, error) {
|
||||
if o.Endpoint == "" {
|
||||
return Options{}, fmt.Errorf("endpoint is required")
|
||||
}
|
||||
if o.Bucket == "" {
|
||||
return Options{}, fmt.Errorf("bucket is required")
|
||||
}
|
||||
if o.Region == "" {
|
||||
o.Region = DefaultRegion
|
||||
}
|
||||
o.Prefix = strings.Trim(o.Prefix, "/")
|
||||
if err := storage.ValidatePrefix(o.Prefix); err != nil {
|
||||
return Options{}, fmt.Errorf("prefix: %w", err)
|
||||
}
|
||||
if (o.AccessKeyID == "") != (o.SecretAccessKey == "") {
|
||||
return Options{}, fmt.Errorf("access key id and secret access key must be configured together")
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
61
internal/adapters/ssh/auth.go
Normal file
61
internal/adapters/ssh/auth.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
|
||||
cryptossh "golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/agent"
|
||||
)
|
||||
|
||||
type agentDialer func(network, address string) (net.Conn, error)
|
||||
|
||||
func authMethods(keyFile string) ([]cryptossh.AuthMethod, func(), error) {
|
||||
return authMethodsWithAgent(os.Getenv("SSH_AUTH_SOCK"), net.Dial, keyFile)
|
||||
}
|
||||
|
||||
func authMethodsWithAgent(agentSocket string, dial agentDialer, keyFile string) ([]cryptossh.AuthMethod, func(), error) {
|
||||
var methods []cryptossh.AuthMethod
|
||||
var closers []io.Closer
|
||||
if agentSocket != "" {
|
||||
methods = append(methods, cryptossh.PublicKeysCallback(func() ([]cryptossh.Signer, error) {
|
||||
conn, err := dial("unix", agentSocket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
closers = append(closers, conn)
|
||||
return agent.NewClient(conn).Signers()
|
||||
}))
|
||||
}
|
||||
if keyFile != "" {
|
||||
signer, err := signerFromKeyFile(keyFile)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
methods = append(methods, cryptossh.PublicKeys(signer))
|
||||
}
|
||||
if len(methods) == 0 {
|
||||
return nil, nil, fmt.Errorf("no SSH auth methods configured; set SSH_AUTH_SOCK or ssh_key_file")
|
||||
}
|
||||
return methods, func() { closeAll(closers) }, nil
|
||||
}
|
||||
|
||||
func signerFromKeyFile(path string) (cryptossh.Signer, error) {
|
||||
key, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read ssh_key_file %q: %w", path, err)
|
||||
}
|
||||
signer, err := cryptossh.ParsePrivateKey(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse ssh_key_file %q: %w", path, err)
|
||||
}
|
||||
return signer, nil
|
||||
}
|
||||
|
||||
func closeAll(closers []io.Closer) {
|
||||
for _, closer := range closers {
|
||||
_ = closer.Close()
|
||||
}
|
||||
}
|
||||
59
internal/adapters/ssh/auth_test.go
Normal file
59
internal/adapters/ssh/auth_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAuthMethodsPreferAgentBeforeKeyFile(t *testing.T) {
|
||||
keyFile := writePrivateKey(t)
|
||||
methods, cleanup, err := authMethodsWithAgent("/tmp/ssh-agent.sock", nil, keyFile)
|
||||
if err != nil {
|
||||
t.Fatalf("authMethodsWithAgent() error = %v", err)
|
||||
}
|
||||
defer cleanup()
|
||||
if got, want := len(methods), 2; got != want {
|
||||
t.Fatalf("auth method count = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthMethodsLoadsKeyFile(t *testing.T) {
|
||||
keyFile := writePrivateKey(t)
|
||||
methods, cleanup, err := authMethodsWithAgent("", nil, keyFile)
|
||||
if err != nil {
|
||||
t.Fatalf("authMethodsWithAgent() error = %v", err)
|
||||
}
|
||||
defer cleanup()
|
||||
if got, want := len(methods), 1; got != want {
|
||||
t.Fatalf("auth method count = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthMethodsRejectsMissingAuth(t *testing.T) {
|
||||
_, _, err := authMethodsWithAgent("", nil, "")
|
||||
if err == nil {
|
||||
t.Fatal("authMethodsWithAgent() error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func writePrivateKey(t *testing.T) string {
|
||||
t.Helper()
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate key: %v", err)
|
||||
}
|
||||
data := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "RSA PRIVATE KEY",
|
||||
Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
|
||||
})
|
||||
path := filepath.Join(t.TempDir(), "id_rsa")
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatalf("write private key: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
521
internal/adapters/ssh/backend.go
Normal file
521
internal/adapters/ssh/backend.go
Normal file
@@ -0,0 +1,521 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"github.com/pkg/sftp"
|
||||
cryptossh "golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type Backend struct {
|
||||
client *sftp.Client
|
||||
sshClient *cryptossh.Client
|
||||
root string
|
||||
}
|
||||
|
||||
func New(ctx context.Context, options Options) (*Backend, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options, err := options.normalized()
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Root, storage.ErrInvalidPath, err)
|
||||
}
|
||||
hostKeyCallback, err := hostKeyCallback(options)
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.KnownHosts, storage.ErrInvalidPath, err)
|
||||
}
|
||||
auth, cleanupAuth, err := authMethods(options.KeyFile)
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.KeyFile, storage.ErrPermission, err)
|
||||
}
|
||||
defer cleanupAuth()
|
||||
|
||||
sshClient, err := cryptossh.Dial("tcp", options.address(), &cryptossh.ClientConfig{
|
||||
User: options.User,
|
||||
Auth: auth,
|
||||
HostKeyCallback: hostKeyCallback,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.address(), storage.ErrUnknown, err)
|
||||
}
|
||||
client, err := sftp.NewClient(sshClient)
|
||||
if err != nil {
|
||||
_ = sshClient.Close()
|
||||
return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.address(), storage.ErrUnknown, err)
|
||||
}
|
||||
return &Backend{client: client, sshClient: sshClient, root: options.Root}, nil
|
||||
}
|
||||
|
||||
func (b *Backend) Close() error {
|
||||
var err error
|
||||
if b.client != nil {
|
||||
err = b.client.Close()
|
||||
}
|
||||
if b.sshClient != nil {
|
||||
if closeErr := b.sshClient.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *Backend) ReadFile(ctx context.Context, logicalPath string) ([]byte, error) {
|
||||
reader, err := b.OpenReader(ctx, logicalPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, storage.NewError(storage.OpReadFile, BackendName, logicalPath, storage.ErrUnknown, err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (b *Backend) OpenReader(ctx context.Context, logicalPath string) (io.ReadCloser, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nativePath, err := b.nativePath(logicalPath, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := b.rejectSymlinkAncestors(ctx, logicalPath, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := b.client.Lstat(nativePath)
|
||||
if err != nil {
|
||||
return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil, storage.NewError(storage.OpOpenReader, BackendName, logicalPath, storage.ErrUnsupported, nil)
|
||||
}
|
||||
file, err := b.client.Open(nativePath)
|
||||
if err != nil {
|
||||
return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (b *Backend) WriteFile(ctx context.Context, logicalPath string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
|
||||
opts.Size = int64(len(data))
|
||||
opts.SizeKnown = true
|
||||
return b.WriteFrom(ctx, logicalPath, bytes.NewReader(data), opts)
|
||||
}
|
||||
|
||||
func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
nativePath, err := b.nativePath(logicalPath, false)
|
||||
if err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
if err := b.rejectSymlinkAncestors(ctx, parentOf(logicalPath), true); err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
if info, err := b.client.Lstat(nativePath); err == nil {
|
||||
if !opts.Overwrite {
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrAlreadyExist, nil)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, nil)
|
||||
}
|
||||
} else if !isNotExist(err) {
|
||||
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
|
||||
}
|
||||
|
||||
parentNative := path.Dir(nativePath)
|
||||
if err := b.client.MkdirAll(parentNative); err != nil {
|
||||
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
|
||||
}
|
||||
|
||||
writePath := nativePath
|
||||
if opts.PreferAtomic {
|
||||
writePath = path.Join(parentNative, fmt.Sprintf(".distributor-write-%d", time.Now().UnixNano()))
|
||||
}
|
||||
file, err := b.client.Create(writePath)
|
||||
if err != nil {
|
||||
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
|
||||
}
|
||||
cleanup := opts.PreferAtomic
|
||||
defer func() {
|
||||
if cleanup {
|
||||
_ = b.client.Remove(writePath)
|
||||
}
|
||||
}()
|
||||
|
||||
written, copyErr := io.Copy(file, r)
|
||||
closeErr := file.Close()
|
||||
if copyErr != nil {
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, closeErr)
|
||||
}
|
||||
if opts.SizeKnown && written != opts.Size {
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", written, opts.Size))
|
||||
}
|
||||
if opts.PreferAtomic {
|
||||
if err := b.client.Rename(writePath, nativePath); err != nil {
|
||||
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
|
||||
}
|
||||
cleanup = false
|
||||
}
|
||||
return b.Stat(ctx, logicalPath)
|
||||
}
|
||||
|
||||
func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
nativePath, err := b.nativePath(logicalPath, true)
|
||||
if err != nil {
|
||||
return storage.Entry{}, err
|
||||
}
|
||||
info, err := b.client.Lstat(nativePath)
|
||||
if err != nil {
|
||||
return storage.Entry{}, b.translateError(storage.OpStat, logicalPath, err)
|
||||
}
|
||||
return entryFromInfo(logicalPath, info), nil
|
||||
}
|
||||
|
||||
func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
nativePrefix, err := b.nativePath(prefix, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := b.client.Lstat(nativePrefix)
|
||||
if err != nil {
|
||||
if isNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return b.translateError(storage.OpWalk, prefix, err)
|
||||
}
|
||||
|
||||
visited := 0
|
||||
emit := func(entry storage.Entry) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Limit > 0 && visited >= opts.Limit {
|
||||
return storage.ErrStopWalk
|
||||
}
|
||||
visited++
|
||||
if err := fn(entry); err != nil {
|
||||
if errors.Is(err, storage.ErrStopWalk) {
|
||||
return storage.ErrStopWalk
|
||||
}
|
||||
return storage.NewError(storage.OpWalk, BackendName, entry.Path, storage.ErrUnknown, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
if err := emit(entryFromInfo(prefix, info)); errors.Is(err, storage.ErrStopWalk) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := b.walkDirectory(ctx, prefix, nativePrefix, opts, emit); errors.Is(err, storage.ErrStopWalk) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||
found := false
|
||||
err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
|
||||
found = true
|
||||
return storage.ErrStopWalk
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, target := range targets {
|
||||
nativePath, err := b.nativePath(target, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if nativePath == b.root {
|
||||
return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrInvalidPath, nil)
|
||||
}
|
||||
info, err := b.client.Lstat(nativePath)
|
||||
if err != nil {
|
||||
if opts.IgnoreMissing && isNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return b.translateError(storage.OpDeleteManagedBundle, target, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrUnsupported, nil)
|
||||
}
|
||||
if err := b.client.Remove(nativePath); err != nil {
|
||||
return b.translateError(storage.OpDeleteManagedBundle, target, err)
|
||||
}
|
||||
if opts.PruneEmptyDirs {
|
||||
b.pruneEmptyParents(parentOf(target))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) DeletePrefix(ctx context.Context, prefix string, opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := storage.ValidatePrefix(prefix); err != nil {
|
||||
return err
|
||||
}
|
||||
var entries []storage.Entry
|
||||
if prefix != "" {
|
||||
entry, err := b.Stat(ctx, prefix)
|
||||
if err != nil {
|
||||
if opts.IgnoreMissing && storage.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if entry.Type != storage.EntryTypeDirectory {
|
||||
return b.deleteEntry(ctx, entry, opts)
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: true}, func(entry storage.Entry) error {
|
||||
entries = append(entries, entry)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if prefix != "" && len(entries) == 1 {
|
||||
if err := b.deleteEntry(ctx, entries[0], opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.PruneEmptyDirs {
|
||||
b.pruneEmptyParents(parentOf(prefix))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return strings.Count(entries[i].Path, "/") > strings.Count(entries[j].Path, "/")
|
||||
})
|
||||
for _, entry := range entries {
|
||||
if entry.Path == "" {
|
||||
continue
|
||||
}
|
||||
if err := b.deleteEntry(ctx, entry, storage.DeleteOptions{IgnoreMissing: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if opts.PruneEmptyDirs {
|
||||
b.pruneEmptyParents(parentOf(prefix))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) deleteEntry(ctx context.Context, entry storage.Entry, opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
nativePath, err := b.nativePath(entry.Path, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var removeErr error
|
||||
if entry.Type == storage.EntryTypeDirectory {
|
||||
removeErr = b.client.RemoveDirectory(nativePath)
|
||||
} else {
|
||||
removeErr = b.client.Remove(nativePath)
|
||||
}
|
||||
if removeErr != nil {
|
||||
if opts.IgnoreMissing && isNotExist(removeErr) {
|
||||
return nil
|
||||
}
|
||||
return b.translateError(storage.OpDeletePrefix, entry.Path, removeErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) walkDirectory(ctx context.Context, logicalPrefix, nativePrefix string, opts storage.WalkOptions, emit func(storage.Entry) error) error {
|
||||
entries, err := b.client.ReadDir(nativePrefix)
|
||||
if err != nil {
|
||||
return b.translateError(storage.OpWalk, logicalPrefix, err)
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
|
||||
for _, info := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
logicalPath := info.Name()
|
||||
if logicalPrefix != "" {
|
||||
logicalPath = logicalPrefix + "/" + info.Name()
|
||||
}
|
||||
if err := emit(entryFromInfo(logicalPath, info)); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Recursive && info.IsDir() {
|
||||
if err := b.walkDirectory(ctx, logicalPath, path.Join(nativePrefix, info.Name()), opts, emit); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) {
|
||||
if logicalPath == "" {
|
||||
if !allowEmpty {
|
||||
return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||
}
|
||||
return b.root, nil
|
||||
}
|
||||
if err := storage.ValidatePath(logicalPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
nativePath := path.Clean(path.Join(b.root, logicalPath))
|
||||
if !withinRoot(b.root, nativePath) {
|
||||
return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||
}
|
||||
return nativePath, nil
|
||||
}
|
||||
|
||||
func (b *Backend) rejectSymlinkAncestors(ctx context.Context, logicalPath string, includeFinal bool) error {
|
||||
if logicalPath == "" {
|
||||
return nil
|
||||
}
|
||||
if err := storage.ValidatePath(logicalPath); err != nil {
|
||||
return err
|
||||
}
|
||||
segments := strings.Split(logicalPath, "/")
|
||||
limit := len(segments)
|
||||
if !includeFinal {
|
||||
limit--
|
||||
}
|
||||
current := ""
|
||||
for index := 0; index < limit; index++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if current == "" {
|
||||
current = segments[index]
|
||||
} else {
|
||||
current += "/" + segments[index]
|
||||
}
|
||||
nativePath, err := b.nativePath(current, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := b.client.Lstat(nativePath)
|
||||
if err != nil {
|
||||
if isNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return b.translateError(storage.OpStat, current, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return storage.NewError(storage.OpStat, BackendName, current, storage.ErrUnsupported, nil)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) pruneEmptyParents(logicalPath string) {
|
||||
for logicalPath != "" {
|
||||
nativePath, err := b.nativePath(logicalPath, false)
|
||||
if err != nil || nativePath == b.root {
|
||||
return
|
||||
}
|
||||
if err := b.client.RemoveDirectory(nativePath); err != nil {
|
||||
return
|
||||
}
|
||||
logicalPath = parentOf(logicalPath)
|
||||
}
|
||||
}
|
||||
|
||||
func withinRoot(root, candidate string) bool {
|
||||
if candidate == root {
|
||||
return true
|
||||
}
|
||||
if root == "/" {
|
||||
return strings.HasPrefix(candidate, "/")
|
||||
}
|
||||
return strings.HasPrefix(candidate, strings.TrimSuffix(root, "/")+"/")
|
||||
}
|
||||
|
||||
func parentOf(logicalPath string) string {
|
||||
index := strings.LastIndex(logicalPath, "/")
|
||||
if index == -1 {
|
||||
return ""
|
||||
}
|
||||
return logicalPath[:index]
|
||||
}
|
||||
|
||||
func isNotExist(err error) bool {
|
||||
return errors.Is(err, fs.ErrNotExist) || errors.Is(err, os.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile)
|
||||
}
|
||||
|
||||
func (b *Backend) translateError(op, logicalPath string, err error) error {
|
||||
kind := storage.ErrUnknown
|
||||
switch {
|
||||
case isNotExist(err):
|
||||
kind = storage.ErrNotFound
|
||||
case errors.Is(err, fs.ErrExist), errors.Is(err, os.ErrExist):
|
||||
kind = storage.ErrAlreadyExist
|
||||
case errors.Is(err, fs.ErrPermission), errors.Is(err, os.ErrPermission), errors.Is(err, sftp.ErrSSHFxPermissionDenied):
|
||||
kind = storage.ErrPermission
|
||||
case errors.Is(err, sftp.ErrSSHFxOpUnsupported):
|
||||
kind = storage.ErrUnsupported
|
||||
case errors.Is(err, sftp.ErrSSHFxNoConnection), errors.Is(err, sftp.ErrSSHFxConnectionLost):
|
||||
kind = storage.ErrTemporary
|
||||
}
|
||||
return storage.NewError(op, BackendName, logicalPath, kind, err)
|
||||
}
|
||||
|
||||
func entryFromInfo(logicalPath string, info fs.FileInfo) storage.Entry {
|
||||
entryType := storage.EntryTypeOther
|
||||
switch {
|
||||
case info.Mode()&os.ModeSymlink != 0:
|
||||
entryType = storage.EntryTypeSymlink
|
||||
case info.Mode().IsRegular():
|
||||
entryType = storage.EntryTypeFile
|
||||
case info.IsDir():
|
||||
entryType = storage.EntryTypeDirectory
|
||||
}
|
||||
return storage.Entry{
|
||||
Path: logicalPath,
|
||||
Type: entryType,
|
||||
Size: info.Size(),
|
||||
}
|
||||
}
|
||||
84
internal/adapters/ssh/hostkeys.go
Normal file
84
internal/adapters/ssh/hostkeys.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
|
||||
cryptossh "golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
)
|
||||
|
||||
func hostKeyCallback(options Options) (cryptossh.HostKeyCallback, error) {
|
||||
switch options.HostKeyPolicy {
|
||||
case HostKeyPolicyOff:
|
||||
return cryptossh.InsecureIgnoreHostKey(), nil
|
||||
case HostKeyPolicyStrict:
|
||||
if options.KnownHosts == "" {
|
||||
return nil, fmt.Errorf("known_hosts is required for strict host key checking")
|
||||
}
|
||||
callback, err := knownhosts.New(options.KnownHosts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load known_hosts %q: %w", options.KnownHosts, err)
|
||||
}
|
||||
return callback, nil
|
||||
case HostKeyPolicyAcceptNew:
|
||||
return acceptNewHostKeyCallback(options)
|
||||
default:
|
||||
return nil, fmt.Errorf("host_key_policy must be strict, accept-new, or off")
|
||||
}
|
||||
}
|
||||
|
||||
func acceptNewHostKeyCallback(options Options) (cryptossh.HostKeyCallback, error) {
|
||||
var checker cryptossh.HostKeyCallback
|
||||
if options.KnownHosts != "" {
|
||||
loaded, err := knownhosts.New(options.KnownHosts)
|
||||
if err == nil {
|
||||
checker = loaded
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("load known_hosts %q: %w", options.KnownHosts, err)
|
||||
}
|
||||
}
|
||||
return func(hostname string, remote net.Addr, key cryptossh.PublicKey) error {
|
||||
if checker != nil {
|
||||
err := checker(hostname, remote, key)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var keyErr *knownhosts.KeyError
|
||||
if !errors.As(err, &keyErr) {
|
||||
return err
|
||||
}
|
||||
if len(keyErr.Want) > 0 {
|
||||
return fmt.Errorf("host key for %s has changed: %w", hostname, err)
|
||||
}
|
||||
}
|
||||
if options.ReadOnlyKnownHosts {
|
||||
return nil
|
||||
}
|
||||
if options.KnownHosts == "" {
|
||||
return fmt.Errorf("host key for %s is unknown and no writable known_hosts path is available", hostname)
|
||||
}
|
||||
if err := appendKnownHost(options.KnownHosts, hostname, key); err != nil {
|
||||
return err
|
||||
}
|
||||
loaded, err := knownhosts.New(options.KnownHosts)
|
||||
if err == nil {
|
||||
checker = loaded
|
||||
}
|
||||
return nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
func appendKnownHost(path, host string, key cryptossh.PublicKey) error {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("persist accepted host key to known_hosts %q: %w", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
if _, err := fmt.Fprintln(file, knownhosts.Line([]string{knownhosts.Normalize(host)}, key)); err != nil {
|
||||
return fmt.Errorf("persist accepted host key to known_hosts %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
138
internal/adapters/ssh/hostkeys_test.go
Normal file
138
internal/adapters/ssh/hostkeys_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
cryptossh "golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
)
|
||||
|
||||
func TestAcceptNewHostKeyCallbackPersistsUnknownHost(t *testing.T) {
|
||||
key := testPublicKey(t)
|
||||
knownHosts := filepath.Join(t.TempDir(), "known_hosts")
|
||||
callback, err := acceptNewHostKeyCallback(Options{KnownHosts: knownHosts})
|
||||
if err != nil {
|
||||
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
|
||||
}
|
||||
|
||||
if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil {
|
||||
t.Fatalf("callback() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(knownHosts)
|
||||
if err != nil {
|
||||
t.Fatalf("read known_hosts: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), "example.com") {
|
||||
t.Fatalf("known_hosts = %q, want example.com entry", data)
|
||||
}
|
||||
if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil {
|
||||
t.Fatalf("second callback() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptNewHostKeyCallbackReadOnlyDoesNotPersistUnknownHost(t *testing.T) {
|
||||
key := testPublicKey(t)
|
||||
knownHosts := filepath.Join(t.TempDir(), "known_hosts")
|
||||
callback, err := acceptNewHostKeyCallback(Options{
|
||||
KnownHosts: knownHosts,
|
||||
ReadOnlyKnownHosts: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
|
||||
}
|
||||
|
||||
if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil {
|
||||
t.Fatalf("callback() error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(knownHosts); !os.IsNotExist(err) {
|
||||
t.Fatalf("known_hosts stat error = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptNewHostKeyCallbackRejectsChangedHostKey(t *testing.T) {
|
||||
first := testPublicKey(t)
|
||||
second := testPublicKey(t)
|
||||
knownHosts := filepath.Join(t.TempDir(), "known_hosts")
|
||||
if err := os.WriteFile(knownHosts, []byte(knownhosts.Line([]string{knownhosts.Normalize("example.com:22")}, first)+"\n"), 0o600); err != nil {
|
||||
t.Fatalf("write known_hosts: %v", err)
|
||||
}
|
||||
callback, err := acceptNewHostKeyCallback(Options{KnownHosts: knownHosts})
|
||||
if err != nil {
|
||||
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
|
||||
}
|
||||
|
||||
err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, second)
|
||||
if err == nil || !strings.Contains(err.Error(), "has changed") {
|
||||
t.Fatalf("callback() error = %v, want changed host key", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptNewHostKeyCallbackReadOnlyRejectsChangedHostKey(t *testing.T) {
|
||||
first := testPublicKey(t)
|
||||
second := testPublicKey(t)
|
||||
knownHosts := filepath.Join(t.TempDir(), "known_hosts")
|
||||
if err := os.WriteFile(knownHosts, []byte(knownhosts.Line([]string{knownhosts.Normalize("example.com:22")}, first)+"\n"), 0o600); err != nil {
|
||||
t.Fatalf("write known_hosts: %v", err)
|
||||
}
|
||||
callback, err := acceptNewHostKeyCallback(Options{
|
||||
KnownHosts: knownHosts,
|
||||
ReadOnlyKnownHosts: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
|
||||
}
|
||||
|
||||
err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, second)
|
||||
if err == nil || !strings.Contains(err.Error(), "has changed") {
|
||||
t.Fatalf("callback() error = %v, want changed host key", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptNewHostKeyCallbackRequiresWritableKnownHostsForUnknownHost(t *testing.T) {
|
||||
callback, err := acceptNewHostKeyCallback(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
|
||||
}
|
||||
|
||||
err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, testPublicKey(t))
|
||||
if err == nil || !strings.Contains(err.Error(), "no writable known_hosts path") {
|
||||
t.Fatalf("callback() error = %v, want no writable known_hosts path", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptNewHostKeyCallbackReadOnlyAllowsMissingKnownHosts(t *testing.T) {
|
||||
callback, err := acceptNewHostKeyCallback(Options{ReadOnlyKnownHosts: true})
|
||||
if err != nil {
|
||||
t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
|
||||
}
|
||||
|
||||
if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, testPublicKey(t)); err != nil {
|
||||
t.Fatalf("callback() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrictHostKeyCallbackRequiresKnownHosts(t *testing.T) {
|
||||
_, err := hostKeyCallback(Options{HostKeyPolicy: HostKeyPolicyStrict})
|
||||
if err == nil || !strings.Contains(err.Error(), "known_hosts is required") {
|
||||
t.Fatalf("hostKeyCallback() error = %v, want known_hosts required", err)
|
||||
}
|
||||
}
|
||||
|
||||
func testPublicKey(t *testing.T) cryptossh.PublicKey {
|
||||
t.Helper()
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("generate key: %v", err)
|
||||
}
|
||||
publicKey, err := cryptossh.NewPublicKey(&privateKey.PublicKey)
|
||||
if err != nil {
|
||||
t.Fatalf("new public key: %v", err)
|
||||
}
|
||||
return publicKey
|
||||
}
|
||||
39
internal/adapters/ssh/integration_test.go
Normal file
39
internal/adapters/ssh/integration_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIntegrationSSHBackendStatRoot(t *testing.T) {
|
||||
host := os.Getenv("DISTRIBUTOR_TEST_SSH_HOST")
|
||||
if host == "" {
|
||||
t.Skip("DISTRIBUTOR_TEST_SSH_HOST is not set")
|
||||
}
|
||||
port := 22
|
||||
if raw := os.Getenv("DISTRIBUTOR_TEST_SSH_PORT"); raw != "" {
|
||||
parsed, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse DISTRIBUTOR_TEST_SSH_PORT: %v", err)
|
||||
}
|
||||
port = parsed
|
||||
}
|
||||
backend, err := New(context.Background(), Options{
|
||||
Host: host,
|
||||
User: os.Getenv("DISTRIBUTOR_TEST_SSH_USER"),
|
||||
Port: port,
|
||||
Root: os.Getenv("DISTRIBUTOR_TEST_SSH_PATH"),
|
||||
KeyFile: os.Getenv("DISTRIBUTOR_TEST_SSH_KEY_FILE"),
|
||||
KnownHosts: os.Getenv("DISTRIBUTOR_TEST_SSH_KNOWN_HOSTS"),
|
||||
HostKeyPolicy: HostKeyPolicyStrict,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
defer backend.Close()
|
||||
if _, err := backend.Stat(context.Background(), ""); err != nil {
|
||||
t.Fatalf("Stat(root) error = %v", err)
|
||||
}
|
||||
}
|
||||
78
internal/adapters/ssh/options.go
Normal file
78
internal/adapters/ssh/options.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const (
|
||||
BackendName = "ssh"
|
||||
|
||||
HostKeyPolicyStrict HostKeyPolicy = "strict"
|
||||
HostKeyPolicyAcceptNew HostKeyPolicy = "accept-new"
|
||||
HostKeyPolicyOff HostKeyPolicy = "off"
|
||||
)
|
||||
|
||||
type HostKeyPolicy string
|
||||
|
||||
type Options struct {
|
||||
Host string
|
||||
User string
|
||||
Port int
|
||||
Root string
|
||||
KeyFile string
|
||||
KnownHosts string
|
||||
HostKeyPolicy HostKeyPolicy
|
||||
ReadOnlyKnownHosts bool
|
||||
}
|
||||
|
||||
func (o Options) normalized() (Options, error) {
|
||||
if o.Host == "" {
|
||||
return Options{}, fmt.Errorf("host is required")
|
||||
}
|
||||
if o.User == "" {
|
||||
current, err := user.Current()
|
||||
if err != nil || current.Username == "" {
|
||||
return Options{}, fmt.Errorf("user is required when current OS user cannot be determined")
|
||||
}
|
||||
o.User = current.Username
|
||||
}
|
||||
if o.Port == 0 {
|
||||
o.Port = 22
|
||||
}
|
||||
if o.Port < 1 || o.Port > 65535 {
|
||||
return Options{}, fmt.Errorf("port must be between 1 and 65535")
|
||||
}
|
||||
if o.Root == "" {
|
||||
return Options{}, fmt.Errorf("path is required")
|
||||
}
|
||||
o.Root = path.Clean(o.Root)
|
||||
if o.HostKeyPolicy == "" {
|
||||
o.HostKeyPolicy = HostKeyPolicyAcceptNew
|
||||
}
|
||||
switch o.HostKeyPolicy {
|
||||
case HostKeyPolicyStrict, HostKeyPolicyAcceptNew, HostKeyPolicyOff:
|
||||
default:
|
||||
return Options{}, fmt.Errorf("host_key_policy must be strict, accept-new, or off")
|
||||
}
|
||||
if o.KnownHosts == "" && o.HostKeyPolicy != HostKeyPolicyOff {
|
||||
o.KnownHosts = defaultKnownHostsPath()
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
func (o Options) address() string {
|
||||
return o.Host + ":" + strconv.Itoa(o.Port)
|
||||
}
|
||||
|
||||
func defaultKnownHostsPath() string {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil || home == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(home, ".ssh", "known_hosts")
|
||||
}
|
||||
118
internal/adapters/ssh/options_test.go
Normal file
118
internal/adapters/ssh/options_test.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"github.com/pkg/sftp"
|
||||
)
|
||||
|
||||
func TestOptionsNormalizeDefaultsUserPortAndHostKeyPolicy(t *testing.T) {
|
||||
options, err := (Options{
|
||||
Host: "example.com",
|
||||
Root: "/reports",
|
||||
}).normalized()
|
||||
if err != nil {
|
||||
t.Fatalf("normalized() error = %v", err)
|
||||
}
|
||||
if options.User == "" {
|
||||
t.Fatal("normalized user is empty")
|
||||
}
|
||||
if options.Port != 22 {
|
||||
t.Fatalf("port = %d, want 22", options.Port)
|
||||
}
|
||||
if options.HostKeyPolicy != HostKeyPolicyAcceptNew {
|
||||
t.Fatalf("host key policy = %q, want accept-new", options.HostKeyPolicy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptionsNormalizeRejectsInvalidFields(t *testing.T) {
|
||||
tests := map[string]Options{
|
||||
"host": {Root: "/reports"},
|
||||
"port": {
|
||||
Host: "example.com",
|
||||
Port: 70000,
|
||||
Root: "/reports",
|
||||
},
|
||||
"path": {
|
||||
Host: "example.com",
|
||||
},
|
||||
"host key policy": {
|
||||
Host: "example.com",
|
||||
Root: "/reports",
|
||||
HostKeyPolicy: "prompt",
|
||||
},
|
||||
}
|
||||
for name, options := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := options.normalized(); err == nil {
|
||||
t.Fatal("normalized() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativePathEnforcesLogicalPathRules(t *testing.T) {
|
||||
backend := &Backend{root: "/srv/reports"}
|
||||
tests := map[string]string{
|
||||
"bundle/report.md": "/srv/reports/bundle/report.md",
|
||||
"": "/srv/reports",
|
||||
}
|
||||
for logicalPath, want := range tests {
|
||||
t.Run(logicalPath, func(t *testing.T) {
|
||||
got, err := backend.nativePath(logicalPath, true)
|
||||
if err != nil {
|
||||
t.Fatalf("nativePath() error = %v", err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("nativePath() = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, logicalPath := range []string{"/absolute", "../escape", "a/../b", `a\b`} {
|
||||
t.Run("reject "+logicalPath, func(t *testing.T) {
|
||||
_, err := backend.nativePath(logicalPath, true)
|
||||
if err == nil || !storage.IsInvalidPath(err) {
|
||||
t.Fatalf("nativePath() error = %v, want invalid path", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRejectsMissingAuthBeforeDial(t *testing.T) {
|
||||
t.Setenv("SSH_AUTH_SOCK", "")
|
||||
_, err := New(context.Background(), Options{
|
||||
Host: "example.com",
|
||||
User: "reports",
|
||||
Root: "/reports",
|
||||
HostKeyPolicy: HostKeyPolicyOff,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "no SSH auth methods configured") {
|
||||
t.Fatalf("New() error = %v, want missing auth", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateErrorMapsSFTPStatusCodes(t *testing.T) {
|
||||
backend := &Backend{}
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want storage.ErrorKind
|
||||
}{
|
||||
{name: "not found", err: sftp.ErrSSHFxNoSuchFile, want: storage.ErrNotFound},
|
||||
{name: "permission", err: sftp.ErrSSHFxPermissionDenied, want: storage.ErrPermission},
|
||||
{name: "unsupported", err: sftp.ErrSSHFxOpUnsupported, want: storage.ErrUnsupported},
|
||||
{name: "temporary", err: sftp.ErrSSHFxConnectionLost, want: storage.ErrTemporary},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := backend.translateError(storage.OpStat, "report.md", tt.err)
|
||||
if !storage.IsKind(err, tt.want) {
|
||||
t.Fatalf("translateError() = %v, want kind %s", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,19 +3,45 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
|
||||
s3adapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/s3"
|
||||
sshadapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/ssh"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
const storagePathKey = "path"
|
||||
|
||||
const (
|
||||
sshHostKey = "host"
|
||||
sshUserKey = "user"
|
||||
sshPortKey = "port"
|
||||
sshKeyFileKey = "ssh_key_file"
|
||||
sshKnownHostsKey = "known_hosts"
|
||||
sshHostKeyPolicyKey = "host_key_policy"
|
||||
sshReadOnlyHostsKey = "read_only_known_hosts"
|
||||
s3EndpointKey = "endpoint"
|
||||
s3BucketKey = "bucket"
|
||||
s3PrefixKey = "prefix"
|
||||
s3RegionKey = "region"
|
||||
s3ForcePathStyleKey = "force_path_style"
|
||||
s3AccessKeyIDKey = "access_key_id"
|
||||
s3SecretAccessKey = "secret_access_key"
|
||||
)
|
||||
|
||||
type backendFactory struct {
|
||||
registry *storage.Registry
|
||||
registry *storage.Registry
|
||||
environment config.Environment
|
||||
readOnlyKnownHosts bool
|
||||
}
|
||||
|
||||
func newBackendFactory() *backendFactory {
|
||||
return newBackendFactoryWithEnvironment(config.ProcessEnvironment())
|
||||
}
|
||||
|
||||
func newBackendFactoryWithEnvironment(environment config.Environment) *backendFactory {
|
||||
registry := storage.NewRegistry()
|
||||
_ = registry.Register(config.BackendLocal, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -23,23 +49,144 @@ func newBackendFactory() *backendFactory {
|
||||
}
|
||||
return local.New(cfg[storagePathKey])
|
||||
})
|
||||
return &backendFactory{registry: registry}
|
||||
_ = registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
port, err := strconv.Atoi(cfg[sshPortKey])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ssh port: %w", err)
|
||||
}
|
||||
readOnlyKnownHosts := false
|
||||
if raw := cfg[sshReadOnlyHostsKey]; raw != "" {
|
||||
readOnlyKnownHosts, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ssh read_only_known_hosts: %w", err)
|
||||
}
|
||||
}
|
||||
return sshadapter.New(ctx, sshadapter.Options{
|
||||
Host: cfg[sshHostKey],
|
||||
User: cfg[sshUserKey],
|
||||
Port: port,
|
||||
Root: cfg[storagePathKey],
|
||||
KeyFile: cfg[sshKeyFileKey],
|
||||
KnownHosts: cfg[sshKnownHostsKey],
|
||||
HostKeyPolicy: sshadapter.HostKeyPolicy(cfg[sshHostKeyPolicyKey]),
|
||||
ReadOnlyKnownHosts: readOnlyKnownHosts,
|
||||
})
|
||||
})
|
||||
_ = registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
forcePathStyle, err := strconv.ParseBool(cfg[s3ForcePathStyleKey])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("s3 force_path_style: %w", err)
|
||||
}
|
||||
return s3adapter.New(ctx, s3adapter.Options{
|
||||
Endpoint: cfg[s3EndpointKey],
|
||||
Bucket: cfg[s3BucketKey],
|
||||
Prefix: cfg[s3PrefixKey],
|
||||
Region: cfg[s3RegionKey],
|
||||
ForcePathStyle: forcePathStyle,
|
||||
AccessKeyID: cfg[s3AccessKeyIDKey],
|
||||
SecretAccessKey: cfg[s3SecretAccessKey],
|
||||
})
|
||||
})
|
||||
return &backendFactory{registry: registry, environment: environment}
|
||||
}
|
||||
|
||||
func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) {
|
||||
if source.Backend != config.BackendLocal {
|
||||
if source.Backend != config.BackendLocal && source.Backend != config.BackendSSH && source.Backend != config.BackendS3 {
|
||||
return nil, fmt.Errorf("source backend %s is not implemented for execution", source.Backend)
|
||||
}
|
||||
return f.registry.Open(ctx, source.Backend, storage.OpenConfig{storagePathKey: source.Path})
|
||||
openConfig, err := f.sourceOpenConfig(source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.registry.Open(ctx, source.Backend, openConfig)
|
||||
}
|
||||
|
||||
func (f *backendFactory) openDestination(ctx context.Context, destination config.Destination) (storage.Backend, error) {
|
||||
if destination.Backend != config.BackendLocal {
|
||||
if destination.Backend != config.BackendLocal && destination.Backend != config.BackendSSH && destination.Backend != config.BackendS3 {
|
||||
return nil, fmt.Errorf("backend %s is not implemented for execution", destination.Backend)
|
||||
}
|
||||
return f.registry.Open(ctx, destination.Backend, storage.OpenConfig{storagePathKey: destination.Path})
|
||||
openConfig, err := f.destinationOpenConfig(destination)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.registry.Open(ctx, destination.Backend, openConfig)
|
||||
}
|
||||
|
||||
func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storage.Backend, error) {
|
||||
return f.registry.Open(ctx, config.BackendLocal, storage.OpenConfig{storagePathKey: path})
|
||||
}
|
||||
|
||||
func (f *backendFactory) resolveCredentials(creds config.Credentials) (config.ResolvedCredentials, error) {
|
||||
return f.environment.ResolveCredentials(creds)
|
||||
}
|
||||
|
||||
func (f *backendFactory) sourceOpenConfig(source config.Backend) (storage.OpenConfig, error) {
|
||||
cfg := sourceOpenConfig(source)
|
||||
if source.Backend == config.BackendS3 {
|
||||
if err := f.addS3Config(cfg, source.Endpoint, source.Bucket, source.Prefix, source.Region, source.ForcePath, source.Creds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if source.Backend == config.BackendSSH {
|
||||
cfg[sshReadOnlyHostsKey] = strconv.FormatBool(f.readOnlyKnownHosts)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (f *backendFactory) destinationOpenConfig(destination config.Destination) (storage.OpenConfig, error) {
|
||||
cfg := destinationOpenConfig(destination)
|
||||
if destination.Backend == config.BackendS3 {
|
||||
if err := f.addS3Config(cfg, destination.Endpoint, destination.Bucket, destination.Prefix, destination.Region, destination.ForcePath, destination.Creds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if destination.Backend == config.BackendSSH {
|
||||
cfg[sshReadOnlyHostsKey] = strconv.FormatBool(f.readOnlyKnownHosts)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (f *backendFactory) addS3Config(cfg storage.OpenConfig, endpoint, bucket, prefix, region string, forcePath *bool, creds config.Credentials) error {
|
||||
cfg[s3EndpointKey] = endpoint
|
||||
cfg[s3BucketKey] = bucket
|
||||
cfg[s3PrefixKey] = prefix
|
||||
cfg[s3RegionKey] = region
|
||||
cfg[s3ForcePathStyleKey] = strconv.FormatBool(config.ForcePathStyle(forcePath))
|
||||
if creds.AccessKeyIDEnv != "" || creds.SecretAccessKeyEnv != "" {
|
||||
resolved, err := f.resolveCredentials(creds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg[s3AccessKeyIDKey] = resolved.AccessKeyID
|
||||
cfg[s3SecretAccessKey] = resolved.SecretAccessKey
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sourceOpenConfig(source config.Backend) storage.OpenConfig {
|
||||
cfg := storage.OpenConfig{storagePathKey: source.Path}
|
||||
if source.Backend == config.BackendSSH {
|
||||
cfg[sshHostKey] = source.Host
|
||||
cfg[sshUserKey] = source.User
|
||||
cfg[sshPortKey] = strconv.Itoa(source.Port)
|
||||
cfg[sshKeyFileKey] = source.SSH.KeyFile
|
||||
cfg[sshKnownHostsKey] = source.SSH.KnownHosts
|
||||
cfg[sshHostKeyPolicyKey] = string(source.SSH.HostKeyPolicy)
|
||||
cfg[sshReadOnlyHostsKey] = "false"
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func destinationOpenConfig(destination config.Destination) storage.OpenConfig {
|
||||
cfg := storage.OpenConfig{storagePathKey: destination.Path}
|
||||
if destination.Backend == config.BackendSSH {
|
||||
cfg[sshHostKey] = destination.Host
|
||||
cfg[sshUserKey] = destination.User
|
||||
cfg[sshPortKey] = strconv.Itoa(destination.Port)
|
||||
cfg[sshKeyFileKey] = destination.SSH.KeyFile
|
||||
cfg[sshKnownHostsKey] = destination.SSH.KnownHosts
|
||||
cfg[sshHostKeyPolicyKey] = string(destination.SSH.HostKeyPolicy)
|
||||
cfg[sshReadOnlyHostsKey] = "false"
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
)
|
||||
|
||||
func TestBackendFactoryOpensLocalSource(t *testing.T) {
|
||||
@@ -47,14 +49,125 @@ func TestBackendFactoryOpensDirectLocalPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendFactoryOpensSSHSourceWithRegisteredOpener(t *testing.T) {
|
||||
factory := &backendFactory{registry: storage.NewRegistry()}
|
||||
var got storage.OpenConfig
|
||||
if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
got = cfg
|
||||
return fake.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
|
||||
backend, err := factory.openSource(context.Background(), config.Backend{
|
||||
Backend: config.BackendSSH,
|
||||
Host: "source.example.com",
|
||||
User: "reports",
|
||||
Port: 22,
|
||||
Path: "/reports",
|
||||
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("openSource() error = %v", err)
|
||||
}
|
||||
if backend == nil {
|
||||
t.Fatal("openSource() backend = nil")
|
||||
}
|
||||
if got[sshHostKey] != "source.example.com" || got[storagePathKey] != "/reports" {
|
||||
t.Fatalf("open config = %#v, want SSH source fields", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendFactoryOpensSSHDestinationWithRegisteredOpener(t *testing.T) {
|
||||
factory := &backendFactory{registry: storage.NewRegistry()}
|
||||
var got storage.OpenConfig
|
||||
if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
got = cfg
|
||||
return fake.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
|
||||
backend, err := factory.openDestination(context.Background(), config.Destination{
|
||||
Backend: config.BackendSSH,
|
||||
Host: "destination.example.com",
|
||||
User: "deploy",
|
||||
Port: 2222,
|
||||
Path: "/archive",
|
||||
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyStrict},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("openDestination() error = %v", err)
|
||||
}
|
||||
if backend == nil {
|
||||
t.Fatal("openDestination() backend = nil")
|
||||
}
|
||||
if got[sshHostKey] != "destination.example.com" || got[sshPortKey] != "2222" || got[sshHostKeyPolicyKey] != "strict" {
|
||||
t.Fatalf("open config = %#v, want SSH destination fields", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendFactorySetsReadOnlyKnownHostsForDryRunSSH(t *testing.T) {
|
||||
factory := &backendFactory{
|
||||
registry: storage.NewRegistry(),
|
||||
readOnlyKnownHosts: true,
|
||||
}
|
||||
var got storage.OpenConfig
|
||||
if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
got = cfg
|
||||
return fake.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
|
||||
_, err := factory.openDestination(context.Background(), config.Destination{
|
||||
Backend: config.BackendSSH,
|
||||
Host: "destination.example.com",
|
||||
User: "deploy",
|
||||
Port: 22,
|
||||
Path: "/archive",
|
||||
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("openDestination() error = %v", err)
|
||||
}
|
||||
if got[sshReadOnlyHostsKey] != "true" {
|
||||
t.Fatalf("open config %s = %q, want true", sshReadOnlyHostsKey, got[sshReadOnlyHostsKey])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendFactoryUsesPersistentKnownHostsByDefault(t *testing.T) {
|
||||
factory := &backendFactory{registry: storage.NewRegistry()}
|
||||
var got storage.OpenConfig
|
||||
if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
got = cfg
|
||||
return fake.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
|
||||
_, err := factory.openDestination(context.Background(), config.Destination{
|
||||
Backend: config.BackendSSH,
|
||||
Host: "destination.example.com",
|
||||
User: "deploy",
|
||||
Port: 22,
|
||||
Path: "/archive",
|
||||
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("openDestination() error = %v", err)
|
||||
}
|
||||
if got[sshReadOnlyHostsKey] != "false" {
|
||||
t.Fatalf("open config %s = %q, want false", sshReadOnlyHostsKey, got[sshReadOnlyHostsKey])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendFactoryRejectsUnsupportedSource(t *testing.T) {
|
||||
factory := newBackendFactory()
|
||||
_, err := factory.openSource(context.Background(), config.Backend{
|
||||
Backend: config.BackendSSH,
|
||||
URI: "ssh://reports@example.com:22",
|
||||
Path: "/reports",
|
||||
Backend: "ftp",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "source backend ssh is not implemented for execution") {
|
||||
if err == nil || !strings.Contains(err.Error(), "source backend ftp is not implemented for execution") {
|
||||
t.Fatalf("openSource() error = %v, want not implemented", err)
|
||||
}
|
||||
}
|
||||
@@ -62,11 +175,156 @@ func TestBackendFactoryRejectsUnsupportedSource(t *testing.T) {
|
||||
func TestBackendFactoryRejectsUnsupportedDestination(t *testing.T) {
|
||||
factory := newBackendFactory()
|
||||
_, err := factory.openDestination(context.Background(), config.Destination{
|
||||
Backend: config.BackendS3,
|
||||
Endpoint: "https://s3.example.com",
|
||||
Bucket: "reports",
|
||||
Backend: "ftp",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "backend s3 is not implemented for execution") {
|
||||
if err == nil || !strings.Contains(err.Error(), "backend ftp is not implemented for execution") {
|
||||
t.Fatalf("openDestination() error = %v, want not implemented", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendFactoryOpensS3DestinationWithRegisteredOpener(t *testing.T) {
|
||||
factory := &backendFactory{
|
||||
registry: storage.NewRegistry(),
|
||||
environment: config.NewEnvironment(nil, func(string) (string, bool) { return "", false }),
|
||||
}
|
||||
var got storage.OpenConfig
|
||||
if err := factory.registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
got = cfg
|
||||
return fake.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
forcePathStyle := false
|
||||
backend, err := factory.openDestination(context.Background(), config.Destination{
|
||||
Backend: config.BackendS3,
|
||||
Endpoint: "https://s3.example.com",
|
||||
Bucket: "reports",
|
||||
Prefix: "archive",
|
||||
Region: config.DefaultS3Region,
|
||||
ForcePath: &forcePathStyle,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("openDestination() error = %v", err)
|
||||
}
|
||||
if backend == nil {
|
||||
t.Fatal("openDestination() backend = nil")
|
||||
}
|
||||
assertOpenConfig(t, got, map[string]string{
|
||||
s3EndpointKey: "https://s3.example.com",
|
||||
s3BucketKey: "reports",
|
||||
s3PrefixKey: "archive",
|
||||
s3RegionKey: config.DefaultS3Region,
|
||||
s3ForcePathStyleKey: "false",
|
||||
})
|
||||
}
|
||||
|
||||
func TestBackendFactoryResolvesS3CredentialsThroughSecretsAwareEnvironment(t *testing.T) {
|
||||
factory := &backendFactory{
|
||||
registry: storage.NewRegistry(),
|
||||
environment: config.NewEnvironment(map[string]string{
|
||||
"ACCESS_KEY_ID": "secret-access",
|
||||
"SECRET_ACCESS_KEY": "secret-secret",
|
||||
}, func(string) (string, bool) { return "", false }),
|
||||
}
|
||||
var got storage.OpenConfig
|
||||
if err := factory.registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
got = cfg
|
||||
return fake.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register() error = %v", err)
|
||||
}
|
||||
forcePathStyle := true
|
||||
_, err := factory.openSource(context.Background(), config.Backend{
|
||||
Backend: config.BackendS3,
|
||||
Endpoint: "https://s3.example.com",
|
||||
Bucket: "reports",
|
||||
Region: config.DefaultS3Region,
|
||||
ForcePath: &forcePathStyle,
|
||||
Creds: config.Credentials{
|
||||
AccessKeyIDEnv: "ACCESS_KEY_ID",
|
||||
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("openSource() error = %v", err)
|
||||
}
|
||||
assertOpenConfig(t, got, map[string]string{
|
||||
s3AccessKeyIDKey: "secret-access",
|
||||
s3SecretAccessKey: "secret-secret",
|
||||
})
|
||||
}
|
||||
|
||||
func TestBackendFactoryResolvesCredentialsThroughEnvironment(t *testing.T) {
|
||||
factory := newBackendFactoryWithEnvironment(config.NewEnvironment(map[string]string{
|
||||
"ACCESS_KEY_ID": "secret-access",
|
||||
"SECRET_ACCESS_KEY": "secret-secret",
|
||||
}, func(string) (string, bool) {
|
||||
return "", false
|
||||
}))
|
||||
|
||||
creds, err := factory.resolveCredentials(config.Credentials{
|
||||
AccessKeyIDEnv: "ACCESS_KEY_ID",
|
||||
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCredentials() error = %v", err)
|
||||
}
|
||||
if creds.AccessKeyID != "secret-access" || creds.SecretAccessKey != "secret-secret" {
|
||||
t.Fatalf("resolved credentials = %#v", creds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) {
|
||||
cfg := sourceOpenConfig(config.Backend{
|
||||
Backend: config.BackendSSH,
|
||||
Host: "source.example.com",
|
||||
User: "reports",
|
||||
Port: 2222,
|
||||
Path: "/reports",
|
||||
SSH: config.SSH{
|
||||
KeyFile: "/home/reports/.ssh/id_ed25519",
|
||||
KnownHosts: "/home/reports/.ssh/known_hosts",
|
||||
HostKeyPolicy: config.HostKeyPolicyStrict,
|
||||
},
|
||||
})
|
||||
|
||||
assertOpenConfig(t, cfg, map[string]string{
|
||||
storagePathKey: "/reports",
|
||||
sshHostKey: "source.example.com",
|
||||
sshUserKey: "reports",
|
||||
sshPortKey: "2222",
|
||||
sshKeyFileKey: "/home/reports/.ssh/id_ed25519",
|
||||
sshKnownHostsKey: "/home/reports/.ssh/known_hosts",
|
||||
sshHostKeyPolicyKey: "strict",
|
||||
})
|
||||
}
|
||||
|
||||
func TestBackendFactoryBuildsSSHDestinationOpenConfig(t *testing.T) {
|
||||
cfg := destinationOpenConfig(config.Destination{
|
||||
Backend: config.BackendSSH,
|
||||
Host: "destination.example.com",
|
||||
User: "deploy",
|
||||
Port: 22,
|
||||
Path: "/srv/archive",
|
||||
SSH: config.SSH{
|
||||
HostKeyPolicy: config.HostKeyPolicyAcceptNew,
|
||||
},
|
||||
})
|
||||
|
||||
assertOpenConfig(t, cfg, map[string]string{
|
||||
storagePathKey: "/srv/archive",
|
||||
sshHostKey: "destination.example.com",
|
||||
sshUserKey: "deploy",
|
||||
sshPortKey: "22",
|
||||
sshHostKeyPolicyKey: "accept-new",
|
||||
})
|
||||
}
|
||||
|
||||
func assertOpenConfig(t *testing.T, got map[string]string, want map[string]string) {
|
||||
t.Helper()
|
||||
for key, wantValue := range want {
|
||||
if gotValue := got[key]; gotValue != wantValue {
|
||||
t.Fatalf("open config %s = %q, want %q", key, gotValue, wantValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
type RunOptions struct {
|
||||
ConfigPath string
|
||||
DryRun bool
|
||||
Force bool
|
||||
Stdout io.Writer
|
||||
Notifier notify.Notifier
|
||||
}
|
||||
@@ -38,13 +39,29 @@ func Run(ctx context.Context, options RunOptions) error {
|
||||
}
|
||||
|
||||
func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error {
|
||||
return runConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
|
||||
}
|
||||
|
||||
type backendFactoryProvider func(config.Environment) *backendFactory
|
||||
|
||||
func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) error {
|
||||
notifier := options.Notifier
|
||||
if notifier == nil {
|
||||
notifier = notify.Noop{}
|
||||
}
|
||||
summary := runSummary{dryRun: options.DryRun}
|
||||
var failures runFailures
|
||||
backends := newBackendFactory()
|
||||
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if options.Stdout != nil {
|
||||
if err := writeSecretConflictWarnings(options.Stdout, secretLoad.Conflicts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
backends := provider(secretLoad.Environment)
|
||||
backends.readOnlyKnownHosts = options.DryRun
|
||||
transforms := newTransformRegistry()
|
||||
if options.Stdout != nil {
|
||||
if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
|
||||
@@ -52,16 +69,23 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
|
||||
}
|
||||
}
|
||||
for _, pipeline := range cfg.Pipelines {
|
||||
if options.Stdout != nil {
|
||||
if err := writeSSHWarnings(options.Stdout, pipeline); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline %s: %w", pipeline.ID, err)
|
||||
return fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||
}
|
||||
bundles, err := bundle.Discover(ctx, sourceBackend, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline %s discover source bundles: %w", pipeline.ID, err)
|
||||
closeBackend(sourceBackend)
|
||||
return fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||
}
|
||||
if options.Stdout != nil {
|
||||
if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
|
||||
closeBackend(sourceBackend)
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -69,13 +93,20 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
|
||||
for _, destination := range pipeline.Destinations {
|
||||
destinationBackend, err := backends.openDestination(ctx, destination)
|
||||
if err != nil {
|
||||
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
summary.recordFailure()
|
||||
if options.Stdout != nil {
|
||||
writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, err)
|
||||
writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, destination.Backend, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
closeDestination := true
|
||||
deferCloseDestination := func() {
|
||||
if closeDestination {
|
||||
closeBackend(destinationBackend)
|
||||
closeDestination = false
|
||||
}
|
||||
}
|
||||
req := publish.Request{
|
||||
PipelineID: pipeline.ID,
|
||||
DestinationID: destination.ID,
|
||||
@@ -88,36 +119,42 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
|
||||
Transformers: transforms,
|
||||
Transfer: destination.Transfer,
|
||||
DistributorVersion: Version,
|
||||
Force: options.Force,
|
||||
}
|
||||
plan, err := publish.Build(ctx, req)
|
||||
if err != nil && plan.DestinationID == "" {
|
||||
plan = publish.Plan{DestinationID: destination.ID, BundlePath: sourceBundle.RootRelativePath}
|
||||
}
|
||||
if options.Stdout != nil {
|
||||
writePlanLine(options.Stdout, plan, err)
|
||||
writePlanLine(options.Stdout, destination.Backend, plan, err)
|
||||
}
|
||||
if err != nil {
|
||||
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
deferCloseDestination()
|
||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
summary.recordFailure()
|
||||
continue
|
||||
}
|
||||
summary.recordPlan(plan.Action)
|
||||
if !options.DryRun {
|
||||
if err := publish.Execute(ctx, req, plan); err != nil {
|
||||
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
deferCloseDestination()
|
||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
summary.recordFailure()
|
||||
continue
|
||||
}
|
||||
if shouldNotify(plan.Action) {
|
||||
if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
|
||||
failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
deferCloseDestination()
|
||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
summary.recordFailure()
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
deferCloseDestination()
|
||||
}
|
||||
}
|
||||
closeBackend(sourceBackend)
|
||||
}
|
||||
if options.Stdout != nil {
|
||||
if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil {
|
||||
@@ -130,7 +167,19 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error
|
||||
return nil
|
||||
}
|
||||
|
||||
func writePlanLine(w io.Writer, plan publish.Plan, planErr error) {
|
||||
type closeableBackend interface {
|
||||
Close() error
|
||||
}
|
||||
|
||||
func closeBackend(backend storage.Backend) {
|
||||
closeable, ok := backend.(closeableBackend)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
_ = closeable.Close()
|
||||
}
|
||||
|
||||
func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
@@ -139,17 +188,17 @@ func writePlanLine(w io.Writer, plan publish.Plan, planErr error) {
|
||||
if destinationID == "" {
|
||||
destinationID = "unknown"
|
||||
}
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, planErr.Error())
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, planErr.Error())
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, plan.Action, outputSummary(plan.Outputs), plan.Reason)
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, plan.Action, outputSummary(plan.Outputs), plan.Reason)
|
||||
}
|
||||
|
||||
func writeErrorLine(w io.Writer, bundlePath, destinationID string, err error) {
|
||||
func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, err.Error())
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, backend, err.Error())
|
||||
}
|
||||
|
||||
func outputSummary(outputs []publish.Output) string {
|
||||
@@ -174,8 +223,33 @@ func destinationSummary(destinations []config.Destination) string {
|
||||
return strings.Join(ids, ",")
|
||||
}
|
||||
|
||||
func writeSecretConflictWarnings(w io.Writer, conflicts []config.SecretConflict) error {
|
||||
for _, conflict := range conflicts {
|
||||
if _, err := fmt.Fprintf(w, "Warning: secret %s ignored because the real environment already has that variable\n", conflict.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeSSHWarnings(w io.Writer, pipeline config.Pipeline) error {
|
||||
if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
||||
if _, err := fmt.Fprintf(w, "Warning: pipeline=%s source host_key_policy=off disables SSH host key checking\n", pipeline.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, destination := range pipeline.Destinations {
|
||||
if destination.Backend == config.BackendSSH && destination.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
||||
if _, err := fmt.Fprintf(w, "Warning: pipeline=%s destination=%s host_key_policy=off disables SSH host key checking\n", pipeline.ID, destination.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func shouldNotify(action publish.Action) bool {
|
||||
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder
|
||||
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
||||
}
|
||||
|
||||
func notifyEvent(plan publish.Plan) notify.Event {
|
||||
@@ -205,6 +279,7 @@ type runSummary struct {
|
||||
planned int
|
||||
publishNew int
|
||||
replaceOlder int
|
||||
forceReplace int
|
||||
skipped int
|
||||
failures int
|
||||
}
|
||||
@@ -216,6 +291,8 @@ func (s *runSummary) recordPlan(action publish.Action) {
|
||||
s.publishNew++
|
||||
case publish.ActionReplaceOlder:
|
||||
s.replaceOlder++
|
||||
case publish.ActionForceReplace:
|
||||
s.forceReplace++
|
||||
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
||||
s.skipped++
|
||||
}
|
||||
@@ -230,12 +307,13 @@ func (s runSummary) Line() string {
|
||||
if s.failures > 0 {
|
||||
status = "failed"
|
||||
}
|
||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d skipped=%d failed=%d dry_run=%t", status, s.planned, s.publishNew, s.replaceOlder, s.skipped, s.failures, s.dryRun)
|
||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun)
|
||||
}
|
||||
|
||||
type runFailure struct {
|
||||
pipelineID string
|
||||
destinationID string
|
||||
backend string
|
||||
bundlePath string
|
||||
err error
|
||||
}
|
||||
@@ -244,10 +322,11 @@ type runFailures struct {
|
||||
items []runFailure
|
||||
}
|
||||
|
||||
func (f *runFailures) add(pipelineID, destinationID, bundlePath string, err error) {
|
||||
func (f *runFailures) add(pipelineID, destinationID, backend, bundlePath string, err error) {
|
||||
f.items = append(f.items, runFailure{
|
||||
pipelineID: pipelineID,
|
||||
destinationID: destinationID,
|
||||
backend: backend,
|
||||
bundlePath: bundlePath,
|
||||
err: err,
|
||||
})
|
||||
@@ -259,7 +338,7 @@ func (f runFailures) Error() string {
|
||||
}
|
||||
parts := make([]string, 0, len(f.items))
|
||||
for _, item := range f.items {
|
||||
parts = append(parts, fmt.Sprintf("pipeline %s destination %s bundle %s: %v", item.pipelineID, item.destinationID, item.bundlePath, item.err))
|
||||
parts = append(parts, fmt.Sprintf("pipeline %s destination %s backend %s bundle %s: %v", item.pipelineID, item.destinationID, item.backend, item.bundlePath, item.err))
|
||||
}
|
||||
return "run failed: " + strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
@@ -10,10 +10,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
@@ -37,8 +40,8 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"Configured pipelines: 1",
|
||||
"- pipeline=reports source=local bundles=1 destinations=archive",
|
||||
"bundle=. destination=archive action=publish_new outputs=report.md,summary.txt",
|
||||
"Final status: ok planned=1 publish_new=1 replace_older=0 skipped=0 failed=0 dry_run=true",
|
||||
"bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt",
|
||||
"Final status: ok planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("Run() output = %q, want substring %q", output, want)
|
||||
@@ -46,6 +49,144 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDryRunUsesReadOnlySSHKnownHosts(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
cfg := config.Config{Pipelines: []config.Pipeline{{
|
||||
ID: "reports",
|
||||
Source: config.Backend{Backend: config.BackendLocal, Path: sourceRoot},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "archive",
|
||||
Backend: config.BackendSSH,
|
||||
Host: "destination.example.com",
|
||||
Path: "/archive",
|
||||
}},
|
||||
}}}
|
||||
config.ApplyDefaults(&cfg)
|
||||
var got storage.OpenConfig
|
||||
provider := func(environment config.Environment) *backendFactory {
|
||||
registry := storage.NewRegistry()
|
||||
if err := registry.Register(config.BackendLocal, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
return local.New(cfg[storagePathKey])
|
||||
}); err != nil {
|
||||
t.Fatalf("register local backend: %v", err)
|
||||
}
|
||||
if err := registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
got = cfg
|
||||
return fake.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register ssh backend: %v", err)
|
||||
}
|
||||
return &backendFactory{registry: registry, environment: environment}
|
||||
}
|
||||
|
||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, provider); err != nil {
|
||||
t.Fatalf("runConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if got[sshReadOnlyHostsKey] != "true" {
|
||||
t.Fatalf("open config %s = %q, want true", sshReadOnlyHostsKey, got[sshReadOnlyHostsKey])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLoadsSecretsBeforeOpeningBackends(t *testing.T) {
|
||||
sourceRoot := filepath.Join(t.TempDir(), "missing-source")
|
||||
destinationRoot := t.TempDir()
|
||||
configPath := writeConfigFile(t, `
|
||||
secrets:
|
||||
directory: `+filepath.Join(t.TempDir(), "missing-secrets")+`
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
`)
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want secrets directory error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load secrets directory") {
|
||||
t.Fatalf("Run() error = %v, want secrets directory error", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "missing-source") {
|
||||
t.Fatalf("Run() error = %v, opened source before loading secrets", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPrintsSecretConflictWarningWithoutValues(t *testing.T) {
|
||||
name := "DISTRIBUTOR_TEST_RUN_SECRET"
|
||||
t.Setenv(name, "process-value")
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
secretsRoot := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(secretsRoot, name), []byte("secret-value\n"), 0o600); err != nil {
|
||||
t.Fatalf("write secret: %v", err)
|
||||
}
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
configPath := writeConfigFile(t, `
|
||||
secrets:
|
||||
directory: `+secretsRoot+`
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: configPath,
|
||||
DryRun: true,
|
||||
Stdout: &stdout,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "secret "+name+" ignored because the real environment already has that variable") {
|
||||
t.Fatalf("stdout = %q, want secret conflict warning", output)
|
||||
}
|
||||
if strings.Contains(output, "process-value") || strings.Contains(output, "secret-value") {
|
||||
t.Fatalf("stdout exposed secret values: %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteSSHWarningsReportsInsecureHostKeyPolicy(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
err := writeSSHWarnings(&stdout, config.Pipeline{
|
||||
ID: "reports",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendSSH,
|
||||
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff},
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "archive",
|
||||
Backend: config.BackendSSH,
|
||||
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff},
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("writeSSHWarnings() error = %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, want := range []string{
|
||||
"pipeline=reports source host_key_policy=off disables SSH host key checking",
|
||||
"pipeline=reports destination=archive host_key_policy=off disables SSH host key checking",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("output = %q, want substring %q", output, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPublishesNewLocalBundle(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -192,11 +333,14 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
|
||||
t.Fatalf("Run() error = %v, want unmanaged failure", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "pipeline reports destination archive-one backend local bundle .") {
|
||||
t.Fatalf("Run() error = %v, want backend context", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, want := range []string{
|
||||
"destination=archive-one action=error",
|
||||
"destination=archive-two action=publish_new",
|
||||
"Final status: failed planned=1 publish_new=1 replace_older=0 skipped=0 failed=1 dry_run=false",
|
||||
"destination=archive-one backend=local action=error",
|
||||
"destination=archive-two backend=local action=publish_new",
|
||||
"Final status: failed planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=1 dry_run=false",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||
@@ -392,6 +536,32 @@ func TestRunFailsOnUnmanagedDestination(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||
Force: true,
|
||||
Stdout: &stdout,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=force_replace") {
|
||||
t.Fatalf("stdout = %q, want force_replace", stdout.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("unmanaged file stat error = %v, want not exist", err)
|
||||
}
|
||||
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
}
|
||||
|
||||
func TestRunFansOutToLocalDestinations(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
firstDestination := t.TempDir()
|
||||
@@ -406,6 +576,137 @@ func TestRunFansOutToLocalDestinations(t *testing.T) {
|
||||
assertFile(t, filepath.Join(secondDestination, "daily", "summary.txt"), "Summary\n")
|
||||
}
|
||||
|
||||
func TestRunFansOutWithDifferentPublishPolicies(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
archiveDestination := t.TempDir()
|
||||
htmlDestination := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeMixedPolicyFanoutConfig(t, sourceRoot, archiveDestination, htmlDestination)})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
assertFile(t, filepath.Join(archiveDestination, "report.md"), "# Report\nSunny.\n")
|
||||
assertFile(t, filepath.Join(archiveDestination, "summary.txt"), "Summary\n")
|
||||
if _, err := os.Stat(filepath.Join(archiveDestination, "report.html")); !os.IsNotExist(err) {
|
||||
t.Fatalf("archive report.html stat error = %v, want not exist", err)
|
||||
}
|
||||
assertFileContains(t, filepath.Join(htmlDestination, "report.html"), "<h1>Report</h1>")
|
||||
if _, err := os.Stat(filepath.Join(htmlDestination, "report.md")); !os.IsNotExist(err) {
|
||||
t.Fatalf("html report.md stat error = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
|
||||
localSourceRoot := t.TempDir()
|
||||
writeSourceBundle(t, localSourceRoot, "", testBundleOptions{})
|
||||
s3Source := fake.New()
|
||||
testutil.WriteFakeSourceBundle(t, s3Source, "", testutil.BundleOptions{})
|
||||
sshSource := fake.New()
|
||||
testutil.WriteFakeSourceBundle(t, sshSource, "", testutil.BundleOptions{})
|
||||
s3Destination := fake.New()
|
||||
sshDestination := fake.New()
|
||||
s3ToLocalDestination := t.TempDir()
|
||||
sshToLocalDestination := t.TempDir()
|
||||
|
||||
cfg := crossBackendConfig(localSourceRoot, s3ToLocalDestination, sshToLocalDestination)
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
t.Fatalf("cross-backend config validation error = %v", err)
|
||||
}
|
||||
provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
|
||||
"s3:source-bucket": s3Source,
|
||||
"s3:destination-bucket": s3Destination,
|
||||
"ssh:/source": sshSource,
|
||||
"ssh:/destination": sshDestination,
|
||||
})
|
||||
|
||||
var dryRunOutput bytes.Buffer
|
||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true, Stdout: &dryRunOutput}, provider); err != nil {
|
||||
t.Fatalf("dry-run error = %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"pipeline=local-to-s3 source=local",
|
||||
"destination=object-archive backend=s3 action=publish_new",
|
||||
"pipeline=s3-to-local source=s3",
|
||||
"destination=local-archive backend=local action=publish_new",
|
||||
"pipeline=local-to-ssh source=local",
|
||||
"destination=ssh-archive backend=ssh action=publish_new",
|
||||
"pipeline=ssh-to-local source=ssh",
|
||||
"Final status: ok planned=4 publish_new=4 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(dryRunOutput.String(), want) {
|
||||
t.Fatalf("dry-run output = %q, want substring %q", dryRunOutput.String(), want)
|
||||
}
|
||||
}
|
||||
if hasAny, err := s3Destination.HasAny(context.Background(), ""); err != nil || hasAny {
|
||||
t.Fatalf("s3 destination after dry-run hasAny=%t err=%v, want empty", hasAny, err)
|
||||
}
|
||||
if entries, err := os.ReadDir(s3ToLocalDestination); err != nil || len(entries) != 0 {
|
||||
t.Fatalf("s3-to-local destination entries = %v err=%v, want empty", entries, err)
|
||||
}
|
||||
|
||||
var publishOutput bytes.Buffer
|
||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &publishOutput}, provider); err != nil {
|
||||
t.Fatalf("publish error = %v", err)
|
||||
}
|
||||
assertFakeFile(t, s3Destination, "report.md", "# Report\nSunny.\n")
|
||||
assertFakeFile(t, sshDestination, "summary.txt", "Summary\n")
|
||||
assertFile(t, filepath.Join(s3ToLocalDestination, "report.md"), "# Report\nSunny.\n")
|
||||
assertFile(t, filepath.Join(sshToLocalDestination, "summary.txt"), "Summary\n")
|
||||
|
||||
var repeatOutput bytes.Buffer
|
||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &repeatOutput}, provider); err != nil {
|
||||
t.Fatalf("repeat error = %v", err)
|
||||
}
|
||||
if got := strings.Count(repeatOutput.String(), "action=skip_same"); got != 4 {
|
||||
t.Fatalf("repeat output = %q, skip_same count = %d, want 4", repeatOutput.String(), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
|
||||
localSourceRoot := t.TempDir()
|
||||
writeSourceBundle(t, localSourceRoot, "bundle", testBundleOptions{})
|
||||
s3Destination := fake.New()
|
||||
sshDestination := fake.New()
|
||||
mustWriteFake(t, s3Destination, "bundle/old.txt", "old")
|
||||
mustWriteFake(t, s3Destination, "bundle-sibling/keep.txt", "keep")
|
||||
mustWriteFake(t, sshDestination, "bundle/old.txt", "old")
|
||||
mustWriteFake(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
||||
cfg := config.Config{Pipelines: []config.Pipeline{{
|
||||
ID: "reports",
|
||||
Source: config.Backend{Backend: config.BackendLocal, Path: localSourceRoot},
|
||||
Destinations: []config.Destination{
|
||||
{
|
||||
ID: "object-archive",
|
||||
Backend: config.BackendS3,
|
||||
Endpoint: "http://s3.test",
|
||||
Bucket: "destination-bucket",
|
||||
},
|
||||
{
|
||||
ID: "ssh-archive",
|
||||
Backend: config.BackendSSH,
|
||||
Host: "ssh.test",
|
||||
Path: "/destination",
|
||||
},
|
||||
},
|
||||
}}}
|
||||
config.ApplyDefaults(&cfg)
|
||||
provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
|
||||
"s3:destination-bucket": s3Destination,
|
||||
"ssh:/destination": sshDestination,
|
||||
})
|
||||
|
||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Force: true}, provider); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
assertFakeFile(t, s3Destination, "bundle/report.md", "# Report\nSunny.\n")
|
||||
assertFakeMissing(t, s3Destination, "bundle/old.txt")
|
||||
assertFakeFile(t, s3Destination, "bundle-sibling/keep.txt", "keep")
|
||||
assertFakeFile(t, sshDestination, "bundle/report.md", "# Report\nSunny.\n")
|
||||
assertFakeMissing(t, sshDestination, "bundle/old.txt")
|
||||
assertFakeFile(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
||||
}
|
||||
|
||||
func TestRunDryRunDoesNotWrite(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -483,6 +784,34 @@ func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestina
|
||||
return testutil.WriteFanoutLocalConfig(t, sourceRoot, firstDestination, secondDestination)
|
||||
}
|
||||
|
||||
func writeMixedPolicyFanoutConfig(t *testing.T, sourceRoot, archiveDestination, htmlDestination string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+archiveDestination+`
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
- id: html
|
||||
backend: local
|
||||
path: `+htmlDestination+`
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
`)
|
||||
}
|
||||
|
||||
func writeConfigFile(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
@@ -524,6 +853,126 @@ func assertFileContains(t *testing.T, path, want string) {
|
||||
}
|
||||
}
|
||||
|
||||
func assertFakeFile(t *testing.T, backend *fake.Backend, path, want string) {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("read fake file %s: %v", path, err)
|
||||
}
|
||||
if got := string(data); got != want {
|
||||
t.Fatalf("%s = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFakeMissing(t *testing.T, backend *fake.Backend, path string) {
|
||||
t.Helper()
|
||||
if _, err := backend.Stat(context.Background(), path); !storage.IsNotFound(err) {
|
||||
t.Fatalf("fake file %s stat error = %v, want not found", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustWriteFake(t *testing.T, backend *fake.Backend, path, data string) {
|
||||
t.Helper()
|
||||
if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
|
||||
t.Fatalf("write fake file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func crossBackendConfig(localSourceRoot, s3ToLocalDestination, sshToLocalDestination string) config.Config {
|
||||
cfg := config.Config{
|
||||
Pipelines: []config.Pipeline{
|
||||
{
|
||||
ID: "local-to-s3",
|
||||
Source: config.Backend{Backend: config.BackendLocal, Path: localSourceRoot},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "object-archive",
|
||||
Backend: config.BackendS3,
|
||||
Endpoint: "http://s3.test",
|
||||
Bucket: "destination-bucket",
|
||||
}},
|
||||
},
|
||||
{
|
||||
ID: "s3-to-local",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendS3,
|
||||
Endpoint: "http://s3.test",
|
||||
Bucket: "source-bucket",
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "local-archive",
|
||||
Backend: config.BackendLocal,
|
||||
Path: s3ToLocalDestination,
|
||||
}},
|
||||
},
|
||||
{
|
||||
ID: "local-to-ssh",
|
||||
Source: config.Backend{Backend: config.BackendLocal, Path: localSourceRoot},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "ssh-archive",
|
||||
Backend: config.BackendSSH,
|
||||
Host: "ssh.test",
|
||||
Path: "/destination",
|
||||
}},
|
||||
},
|
||||
{
|
||||
ID: "ssh-to-local",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendSSH,
|
||||
Host: "ssh.test",
|
||||
Path: "/source",
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "local-archive",
|
||||
Backend: config.BackendLocal,
|
||||
Path: sshToLocalDestination,
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func fakeBackendFactoryProvider(t *testing.T, remoteBackends map[string]storage.Backend) backendFactoryProvider {
|
||||
t.Helper()
|
||||
return func(environment config.Environment) *backendFactory {
|
||||
registry := storage.NewRegistry()
|
||||
if err := registry.Register(config.BackendLocal, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return local.New(cfg[storagePathKey])
|
||||
}); err != nil {
|
||||
t.Fatalf("register local backend: %v", err)
|
||||
}
|
||||
if err := registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
backend := remoteBackends["s3:"+cfg[s3BucketKey]]
|
||||
if backend == nil {
|
||||
return nil, fmt.Errorf("missing fake s3 backend for bucket %s", cfg[s3BucketKey])
|
||||
}
|
||||
return backend, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register s3 backend: %v", err)
|
||||
}
|
||||
if err := registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
backend := remoteBackends["ssh:"+cfg[storagePathKey]]
|
||||
if backend == nil {
|
||||
return nil, fmt.Errorf("missing fake ssh backend for path %s", cfg[storagePathKey])
|
||||
}
|
||||
return backend, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register ssh backend: %v", err)
|
||||
}
|
||||
return &backendFactory{registry: registry, environment: environment}
|
||||
}
|
||||
}
|
||||
|
||||
type recordingNotifier struct {
|
||||
events []notify.Event
|
||||
check func()
|
||||
|
||||
@@ -180,6 +180,32 @@ func TestExecuteRunDryRun(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
}
|
||||
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := Execute(context.Background(), []string{"run", "--config", configPath, "--force", "--dry-run"}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=force_replace") {
|
||||
t.Fatalf("stdout = %q, want force_replace", stdout.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); err != nil {
|
||||
t.Fatalf("unmanaged file stat error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(err) {
|
||||
t.Fatalf("state stat error = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunRejectsExtraPositionalArgs(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
||||
flags.SetOutput(stderr)
|
||||
configPath := flags.String("config", "", "path to config file")
|
||||
dryRun := flags.Bool("dry-run", false, "load and validate config without publishing")
|
||||
force := flags.Bool("force", false, "allow explicit destructive replacement for supported conflicts")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
@@ -29,6 +30,7 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
||||
if err := app.Run(ctx, app.RunOptions{
|
||||
ConfigPath: *configPath,
|
||||
DryRun: *dryRun,
|
||||
Force: *force,
|
||||
Stdout: stdout,
|
||||
}); err != nil {
|
||||
return fail(stderr, err)
|
||||
@@ -38,13 +40,14 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
||||
|
||||
func printRunHelp(w io.Writer) {
|
||||
fmt.Fprint(w, `Usage:
|
||||
distributor run --config <path> --dry-run
|
||||
distributor run --config <path> [--dry-run] [--force]
|
||||
|
||||
Options:
|
||||
--config <path> Path to config file
|
||||
--dry-run Load and validate config without publishing
|
||||
--force Allow explicit destructive replacement for supported conflicts
|
||||
|
||||
Run discovers local source bundles, plans each configured destination, publishes
|
||||
Run discovers configured source bundles, plans each destination, publishes
|
||||
selected outputs unless --dry-run is set, and prints a final status summary.
|
||||
`)
|
||||
}
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
package config
|
||||
|
||||
type Config struct {
|
||||
Secrets Secrets `yaml:"secrets"`
|
||||
Pipelines []Pipeline `yaml:"pipelines"`
|
||||
}
|
||||
|
||||
type Secrets struct {
|
||||
Directory string `yaml:"directory"`
|
||||
}
|
||||
|
||||
type Pipeline struct {
|
||||
ID string `yaml:"id"`
|
||||
Source Backend `yaml:"source"`
|
||||
@@ -14,14 +19,17 @@ type Pipeline struct {
|
||||
type Destination struct {
|
||||
ID string `yaml:"id"`
|
||||
Backend string `yaml:"backend"`
|
||||
Host string `yaml:"host"`
|
||||
User string `yaml:"user"`
|
||||
Port int `yaml:"port"`
|
||||
Path string `yaml:"path"`
|
||||
URI string `yaml:"uri"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Region string `yaml:"region"`
|
||||
ForcePath bool `yaml:"force_path_style"`
|
||||
ForcePath *bool `yaml:"force_path_style"`
|
||||
Creds Credentials `yaml:"credentials"`
|
||||
SSH SSH `yaml:",inline"`
|
||||
Publish *PublishPolicy `yaml:"publish"`
|
||||
Transform Transform `yaml:"transform"`
|
||||
Transfer TransferPolicy `yaml:"transfer"`
|
||||
@@ -29,14 +37,23 @@ type Destination struct {
|
||||
|
||||
type Backend struct {
|
||||
Backend string `yaml:"backend"`
|
||||
Host string `yaml:"host"`
|
||||
User string `yaml:"user"`
|
||||
Port int `yaml:"port"`
|
||||
Path string `yaml:"path"`
|
||||
URI string `yaml:"uri"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Region string `yaml:"region"`
|
||||
ForcePath bool `yaml:"force_path_style"`
|
||||
ForcePath *bool `yaml:"force_path_style"`
|
||||
Creds Credentials `yaml:"credentials"`
|
||||
SSH SSH `yaml:",inline"`
|
||||
}
|
||||
|
||||
type SSH struct {
|
||||
KeyFile string `yaml:"ssh_key_file"`
|
||||
KnownHosts string `yaml:"known_hosts"`
|
||||
HostKeyPolicy HostKeyPolicy `yaml:"host_key_policy"`
|
||||
}
|
||||
|
||||
type Credentials struct {
|
||||
|
||||
@@ -22,14 +22,18 @@ const (
|
||||
TransformModeSidecar = "sidecar"
|
||||
)
|
||||
|
||||
const DefaultS3Region = "us-east-1"
|
||||
|
||||
func ApplyDefaults(cfg *Config) {
|
||||
for pipelineIndex := range cfg.Pipelines {
|
||||
pipeline := &cfg.Pipelines[pipelineIndex]
|
||||
applyBackendDefaults(&pipeline.Source)
|
||||
if pipeline.Validation.OnDigestMismatch == "" {
|
||||
pipeline.Validation.OnDigestMismatch = ValidationActionFail
|
||||
}
|
||||
for destinationIndex := range pipeline.Destinations {
|
||||
destination := &pipeline.Destinations[destinationIndex]
|
||||
applyDestinationDefaults(destination)
|
||||
if destination.Publish == nil {
|
||||
destination.Publish = &PublishPolicy{Source: true}
|
||||
}
|
||||
@@ -48,3 +52,42 @@ func ApplyDefaults(cfg *Config) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func applyBackendDefaults(backend *Backend) {
|
||||
if backend.Backend == BackendSSH {
|
||||
if backend.Port == 0 {
|
||||
backend.Port = 22
|
||||
}
|
||||
if backend.SSH.HostKeyPolicy == "" {
|
||||
backend.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew
|
||||
}
|
||||
}
|
||||
if backend.Backend == BackendS3 {
|
||||
applyS3Defaults(&backend.Region, &backend.Prefix, &backend.ForcePath)
|
||||
}
|
||||
}
|
||||
|
||||
func applyDestinationDefaults(destination *Destination) {
|
||||
if destination.Backend == BackendSSH {
|
||||
if destination.Port == 0 {
|
||||
destination.Port = 22
|
||||
}
|
||||
if destination.SSH.HostKeyPolicy == "" {
|
||||
destination.SSH.HostKeyPolicy = HostKeyPolicyAcceptNew
|
||||
}
|
||||
}
|
||||
if destination.Backend == BackendS3 {
|
||||
applyS3Defaults(&destination.Region, &destination.Prefix, &destination.ForcePath)
|
||||
}
|
||||
}
|
||||
|
||||
func applyS3Defaults(region, prefix *string, forcePath **bool) {
|
||||
if *region == "" {
|
||||
*region = DefaultS3Region
|
||||
}
|
||||
*prefix = NormalizeS3Prefix(*prefix)
|
||||
if *forcePath == nil {
|
||||
defaultForcePath := true
|
||||
*forcePath = &defaultForcePath
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,29 @@ pipelines:
|
||||
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
|
||||
t.Fatalf("transfer default = %q, want %q", got, want)
|
||||
}
|
||||
if cfg.Secrets.Directory != "" {
|
||||
t.Fatalf("secrets.directory = %q, want empty", cfg.Secrets.Directory)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileValidSecretsDirectoryConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
secrets:
|
||||
directory: /run/secrets/distributor
|
||||
pipelines:
|
||||
- id: local-copy
|
||||
source:
|
||||
backend: local
|
||||
path: /var/spool/reports
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /srv/archive
|
||||
`)
|
||||
|
||||
if got, want := cfg.Secrets.Directory, "/run/secrets/distributor"; got != want {
|
||||
t.Fatalf("secrets.directory = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileValidFanOutConfig(t *testing.T) {
|
||||
@@ -53,7 +76,9 @@ pipelines:
|
||||
html: false
|
||||
- id: static-site
|
||||
backend: ssh
|
||||
uri: ssh://deploy@example.com:22
|
||||
host: example.com
|
||||
user: deploy
|
||||
port: 22
|
||||
path: /srv/www/reports
|
||||
publish:
|
||||
source: false
|
||||
@@ -87,13 +112,19 @@ pipelines:
|
||||
- id: ssh-backend
|
||||
source:
|
||||
backend: ssh
|
||||
uri: ssh://reports@example.com:22
|
||||
host: source.example.com
|
||||
user: reports
|
||||
path: /source
|
||||
destinations:
|
||||
- id: ssh-destination
|
||||
backend: ssh
|
||||
uri: ssh://deploy@example.com:22
|
||||
host: destination.example.com
|
||||
user: deploy
|
||||
port: 2222
|
||||
path: /destination
|
||||
ssh_key_file: /home/deploy/.ssh/id_ed25519
|
||||
known_hosts: /home/deploy/.ssh/known_hosts
|
||||
host_key_policy: strict
|
||||
`,
|
||||
"s3": `
|
||||
pipelines:
|
||||
@@ -124,6 +155,61 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsS3Config(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: s3-defaults
|
||||
source:
|
||||
backend: s3
|
||||
endpoint: http://127.0.0.1:9000
|
||||
bucket: source
|
||||
prefix: /incoming/reports/
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: s3
|
||||
endpoint: http://127.0.0.1:9000
|
||||
bucket: destination
|
||||
`)
|
||||
|
||||
source := cfg.Pipelines[0].Source
|
||||
if got, want := source.Region, DefaultS3Region; got != want {
|
||||
t.Fatalf("source region = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.Prefix, "incoming/reports"; got != want {
|
||||
t.Fatalf("source prefix = %q, want %q", got, want)
|
||||
}
|
||||
if !ForcePathStyle(source.ForcePath) {
|
||||
t.Fatal("source force_path_style = false, want true")
|
||||
}
|
||||
destination := cfg.Pipelines[0].Destinations[0]
|
||||
if got, want := destination.Region, DefaultS3Region; got != want {
|
||||
t.Fatalf("destination region = %q, want %q", got, want)
|
||||
}
|
||||
if !ForcePathStyle(destination.ForcePath) {
|
||||
t.Fatal("destination force_path_style = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFilePreservesExplicitS3ForcePathStyleFalse(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: s3-force-path
|
||||
source:
|
||||
backend: s3
|
||||
endpoint: https://s3.example.com
|
||||
bucket: source
|
||||
force_path_style: false
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`)
|
||||
|
||||
if ForcePathStyle(cfg.Pipelines[0].Source.ForcePath) {
|
||||
t.Fatal("force_path_style = true, want explicit false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsDuplicatePipelineIDs(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
pipelines:
|
||||
@@ -171,7 +257,7 @@ func TestLoadFileRejectsMissingRequiredFields(t *testing.T) {
|
||||
"destinations": `pipelines: [{id: reports, source: {backend: local, path: /source}}]`,
|
||||
"destination id": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{backend: local, path: /archive}]}]`,
|
||||
"local path": `pipelines: [{id: reports, source: {backend: local}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"ssh uri": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"ssh host": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"s3 bucket": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com"}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"publish outputs": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, publish: {source: false, html: false}}]}]`,
|
||||
}
|
||||
@@ -183,6 +269,101 @@ func TestLoadFileRejectsMissingRequiredFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidS3Config(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"prefix traversal": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com", bucket: source, prefix: "../reports"}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"prefix backslash": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com", bucket: source, prefix: 'a\b'}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"partial creds": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com", bucket: source, credentials: {access_key_id_env: ACCESS_KEY_ID}}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assertLoadError(t, body, "")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsSSHConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: ssh-defaults
|
||||
source:
|
||||
backend: ssh
|
||||
host: source.example.com
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: ssh
|
||||
host: destination.example.com
|
||||
path: /archive
|
||||
host_key_policy: false
|
||||
`)
|
||||
|
||||
source := cfg.Pipelines[0].Source
|
||||
if source.Port != 22 {
|
||||
t.Fatalf("source port = %d, want 22", source.Port)
|
||||
}
|
||||
if source.SSH.HostKeyPolicy != HostKeyPolicyAcceptNew {
|
||||
t.Fatalf("source host key policy = %q, want accept-new", source.SSH.HostKeyPolicy)
|
||||
}
|
||||
destination := cfg.Pipelines[0].Destinations[0]
|
||||
if destination.Port != 22 {
|
||||
t.Fatalf("destination port = %d, want 22", destination.Port)
|
||||
}
|
||||
if destination.SSH.HostKeyPolicy != HostKeyPolicyOff {
|
||||
t.Fatalf("destination host key policy = %q, want off", destination.SSH.HostKeyPolicy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileNormalizesSSHHostKeyPolicies(t *testing.T) {
|
||||
tests := map[string]HostKeyPolicy{
|
||||
`true`: HostKeyPolicyStrict,
|
||||
`"true"`: HostKeyPolicyStrict,
|
||||
`strict`: HostKeyPolicyStrict,
|
||||
`accept-new`: HostKeyPolicyAcceptNew,
|
||||
`false`: HostKeyPolicyOff,
|
||||
`"false"`: HostKeyPolicyOff,
|
||||
`off`: HostKeyPolicyOff,
|
||||
`"STRICT"`: HostKeyPolicyStrict,
|
||||
`"ACCEPT-NEW"`: HostKeyPolicyAcceptNew,
|
||||
`"OFF"`: HostKeyPolicyOff,
|
||||
}
|
||||
for value, want := range tests {
|
||||
t.Run(value, func(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: ssh-policy
|
||||
source:
|
||||
backend: ssh
|
||||
host: source.example.com
|
||||
path: /source
|
||||
host_key_policy: `+value+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`)
|
||||
if got := cfg.Pipelines[0].Source.SSH.HostKeyPolicy; got != want {
|
||||
t.Fatalf("host key policy = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsLegacySSHURIFieldAsUnknown(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: ssh
|
||||
uri: ssh://reports@example.com:22
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`, "field uri not found")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsUnsupportedBackend(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
pipelines:
|
||||
@@ -261,12 +442,31 @@ pipelines:
|
||||
`, "field surprise not found")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsUnknownSecretsFields(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
secrets:
|
||||
directory: /run/secrets/distributor
|
||||
surprise: true
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`, "field surprise not found")
|
||||
}
|
||||
|
||||
func TestExampleConfigsLoad(t *testing.T) {
|
||||
for _, path := range []string{
|
||||
"../../examples/local-to-local.yml",
|
||||
"../../examples/local-publish.yml",
|
||||
"../../examples/local-html.yml",
|
||||
"../../examples/fan-out.yml",
|
||||
"../../examples/ssh-destination.yml",
|
||||
"../../examples/s3-destination.yml",
|
||||
} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
if _, err := LoadFile(path); err != nil {
|
||||
|
||||
19
internal/config/s3.go
Normal file
19
internal/config/s3.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
func NormalizeS3Prefix(prefix string) string {
|
||||
return strings.Trim(prefix, "/")
|
||||
}
|
||||
|
||||
func ValidateS3Prefix(prefix string) error {
|
||||
return storage.ValidatePrefix(prefix)
|
||||
}
|
||||
|
||||
func ForcePathStyle(value *bool) bool {
|
||||
return value == nil || *value
|
||||
}
|
||||
154
internal/config/secrets.go
Normal file
154
internal/config/secrets.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var secretNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type EnvLookup func(string) (string, bool)
|
||||
|
||||
type Environment struct {
|
||||
values map[string]string
|
||||
lookup EnvLookup
|
||||
}
|
||||
|
||||
type SecretConflict struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
type SecretLoadResult struct {
|
||||
Environment Environment
|
||||
Conflicts []SecretConflict
|
||||
}
|
||||
|
||||
type ResolvedCredentials struct {
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
}
|
||||
|
||||
func ProcessEnvironment() Environment {
|
||||
return NewEnvironment(nil, os.LookupEnv)
|
||||
}
|
||||
|
||||
func NewEnvironment(values map[string]string, lookup EnvLookup) Environment {
|
||||
copied := make(map[string]string, len(values))
|
||||
for key, value := range values {
|
||||
copied[key] = value
|
||||
}
|
||||
if lookup == nil {
|
||||
lookup = os.LookupEnv
|
||||
}
|
||||
return Environment{values: copied, lookup: lookup}
|
||||
}
|
||||
|
||||
func (e Environment) Lookup(name string) (string, bool) {
|
||||
if e.lookup != nil {
|
||||
if value, ok := e.lookup(name); ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
value, ok := e.values[name]
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func (e Environment) ResolveCredentials(creds Credentials) (ResolvedCredentials, error) {
|
||||
var resolved ResolvedCredentials
|
||||
var err error
|
||||
if creds.AccessKeyIDEnv != "" {
|
||||
resolved.AccessKeyID, err = e.required(creds.AccessKeyIDEnv)
|
||||
if err != nil {
|
||||
return ResolvedCredentials{}, err
|
||||
}
|
||||
}
|
||||
if creds.SecretAccessKeyEnv != "" {
|
||||
resolved.SecretAccessKey, err = e.required(creds.SecretAccessKeyEnv)
|
||||
if err != nil {
|
||||
return ResolvedCredentials{}, err
|
||||
}
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (e Environment) required(name string) (string, error) {
|
||||
value, ok := e.Lookup(name)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("credential environment variable %s is not set", name)
|
||||
}
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("credential environment variable %s is empty", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func LoadSecretEnvironment(directory string, lookup EnvLookup) (SecretLoadResult, error) {
|
||||
if directory == "" {
|
||||
return SecretLoadResult{Environment: NewEnvironment(nil, lookup)}, nil
|
||||
}
|
||||
values, err := loadSecretValues(directory)
|
||||
if err != nil {
|
||||
return SecretLoadResult{}, err
|
||||
}
|
||||
var conflicts []SecretConflict
|
||||
for name, value := range values {
|
||||
if processValue, ok := lookupValue(lookup, name); ok && processValue != value {
|
||||
conflicts = append(conflicts, SecretConflict{Name: name})
|
||||
}
|
||||
}
|
||||
sort.Slice(conflicts, func(i, j int) bool {
|
||||
return conflicts[i].Name < conflicts[j].Name
|
||||
})
|
||||
return SecretLoadResult{
|
||||
Environment: NewEnvironment(values, lookup),
|
||||
Conflicts: conflicts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func loadSecretValues(directory string) (map[string]string, error) {
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load secrets directory %q: %w", directory, err)
|
||||
}
|
||||
values := make(map[string]string)
|
||||
for _, entry := range entries {
|
||||
path := filepath.Join(directory, entry.Name())
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect secret file %q: %w", entry.Name(), err)
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
if !secretNamePattern.MatchString(entry.Name()) {
|
||||
return nil, fmt.Errorf("secret filename %q is invalid", entry.Name())
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read secret file %q: %w", entry.Name(), err)
|
||||
}
|
||||
values[entry.Name()] = trimOneTrailingLineEnding(string(data))
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func trimOneTrailingLineEnding(value string) string {
|
||||
if strings.HasSuffix(value, "\r\n") {
|
||||
return strings.TrimSuffix(value, "\r\n")
|
||||
}
|
||||
if strings.HasSuffix(value, "\n") {
|
||||
return strings.TrimSuffix(value, "\n")
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func lookupValue(lookup EnvLookup, name string) (string, bool) {
|
||||
if lookup == nil {
|
||||
lookup = os.LookupEnv
|
||||
}
|
||||
return lookup(name)
|
||||
}
|
||||
248
internal/config/secrets_test.go
Normal file
248
internal/config/secrets_test.go
Normal file
@@ -0,0 +1,248 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadSecretEnvironmentLoadsValidFiles(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, "API_KEY", "value\n")
|
||||
writeSecret(t, directory, "CRLF", "value\r\n")
|
||||
writeSecret(t, directory, "MULTILINE", "value\n\n")
|
||||
writeSecret(t, directory, "SPACES", " value \n")
|
||||
writeSecret(t, directory, "CARRIAGE", "value\r")
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, emptyLookup)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
assertEnvValue(t, result.Environment, "API_KEY", "value")
|
||||
assertEnvValue(t, result.Environment, "CRLF", "value")
|
||||
assertEnvValue(t, result.Environment, "MULTILINE", "value\n")
|
||||
assertEnvValue(t, result.Environment, "SPACES", " value ")
|
||||
assertEnvValue(t, result.Environment, "CARRIAGE", "value\r")
|
||||
}
|
||||
|
||||
func TestLoadSecretEnvironmentRejectsInvalidFilenames(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
secretValue := "do-not-print"
|
||||
writeSecret(t, directory, "1INVALID", secretValue)
|
||||
|
||||
_, err := LoadSecretEnvironment(directory, emptyLookup)
|
||||
if err == nil {
|
||||
t.Fatal("LoadSecretEnvironment() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "secret filename") {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %q, want filename error", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), secretValue) {
|
||||
t.Fatalf("LoadSecretEnvironment() error exposed secret value: %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretEnvironmentIgnoresDirectoriesAndFollowsSymlinks(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(directory, "IGNORED_DIR"), 0o700); err != nil {
|
||||
t.Fatalf("mkdir ignored dir: %v", err)
|
||||
}
|
||||
targetFile := filepath.Join(t.TempDir(), "target")
|
||||
if err := os.WriteFile(targetFile, []byte("linked\n"), 0o600); err != nil {
|
||||
t.Fatalf("write target file: %v", err)
|
||||
}
|
||||
if err := os.Symlink(targetFile, filepath.Join(directory, "LINKED_SECRET")); err != nil {
|
||||
t.Fatalf("symlink file: %v", err)
|
||||
}
|
||||
targetDir := t.TempDir()
|
||||
if err := os.Symlink(targetDir, filepath.Join(directory, "LINKED_DIR")); err != nil {
|
||||
t.Fatalf("symlink dir: %v", err)
|
||||
}
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, emptyLookup)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
assertEnvValue(t, result.Environment, "LINKED_SECRET", "linked")
|
||||
if _, ok := result.Environment.Lookup("IGNORED_DIR"); ok {
|
||||
t.Fatal("directory appeared in environment")
|
||||
}
|
||||
if _, ok := result.Environment.Lookup("LINKED_DIR"); ok {
|
||||
t.Fatal("directory symlink appeared in environment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretEnvironmentMissingDirectoryFails(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "missing")
|
||||
_, err := LoadSecretEnvironment(missing, emptyLookup)
|
||||
if err == nil {
|
||||
t.Fatal("LoadSecretEnvironment() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load secrets directory") || !strings.Contains(err.Error(), missing) {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %q, want directory context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvironmentPrefersProcessValuesAndReportsDifferingConflicts(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
secretValue := "secret-value"
|
||||
processValue := "process-value"
|
||||
writeSecret(t, directory, "TOKEN", secretValue)
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"TOKEN": processValue}))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
assertEnvValue(t, result.Environment, "TOKEN", processValue)
|
||||
if got, want := len(result.Conflicts), 1; got != want {
|
||||
t.Fatalf("conflict count = %d, want %d", got, want)
|
||||
}
|
||||
if result.Conflicts[0].Name != "TOKEN" {
|
||||
t.Fatalf("conflict name = %q, want TOKEN", result.Conflicts[0].Name)
|
||||
}
|
||||
if strings.Contains(result.Conflicts[0].Name, secretValue) || strings.Contains(result.Conflicts[0].Name, processValue) {
|
||||
t.Fatalf("conflict exposed secret values: %#v", result.Conflicts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvironmentDoesNotWarnWhenProcessValueMatchesSecret(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, "TOKEN", "same-value")
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"TOKEN": "same-value"}))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
if len(result.Conflicts) != 0 {
|
||||
t.Fatalf("conflicts = %#v, want none", result.Conflicts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvironmentDoesNotMutateProcessEnvironment(t *testing.T) {
|
||||
name := "DISTRIBUTOR_TEST_SECRET_ONLY"
|
||||
t.Setenv(name, "")
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatalf("unset env: %v", err)
|
||||
}
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, name, "secret")
|
||||
|
||||
if _, err := LoadSecretEnvironment(directory, nil); err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
if _, ok := os.LookupEnv(name); ok {
|
||||
t.Fatalf("%s was added to process environment", name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsUsesSecretsAwareEnvironment(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, "ACCESS_KEY_ID", "secret-access")
|
||||
writeSecret(t, directory, "SECRET_ACCESS_KEY", "secret-secret")
|
||||
result, err := LoadSecretEnvironment(directory, emptyLookup)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
|
||||
creds, err := result.Environment.ResolveCredentials(Credentials{
|
||||
AccessKeyIDEnv: "ACCESS_KEY_ID",
|
||||
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveCredentials() error = %v", err)
|
||||
}
|
||||
if creds.AccessKeyID != "secret-access" || creds.SecretAccessKey != "secret-secret" {
|
||||
t.Fatalf("resolved credentials = %#v", creds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsPrefersProcessEnvironment(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, "ACCESS_KEY_ID", "secret-access")
|
||||
result, err := LoadSecretEnvironment(directory, mapLookup(map[string]string{"ACCESS_KEY_ID": "process-access"}))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
|
||||
creds, err := result.Environment.ResolveCredentials(Credentials{AccessKeyIDEnv: "ACCESS_KEY_ID"})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveCredentials() error = %v", err)
|
||||
}
|
||||
if creds.AccessKeyID != "process-access" {
|
||||
t.Fatalf("access key = %q, want process-access", creds.AccessKeyID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsDoesNotFeedProcessEnvironment(t *testing.T) {
|
||||
name := "DISTRIBUTOR_TEST_SDK_CHAIN_VALUE"
|
||||
t.Setenv(name, "")
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatalf("unset env: %v", err)
|
||||
}
|
||||
directory := t.TempDir()
|
||||
writeSecret(t, directory, name, "secret")
|
||||
|
||||
result, err := LoadSecretEnvironment(directory, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSecretEnvironment() error = %v", err)
|
||||
}
|
||||
assertEnvValue(t, result.Environment, name, "secret")
|
||||
if _, ok := os.LookupEnv(name); ok {
|
||||
t.Fatalf("%s is visible to process environment", name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsMissingReferenceFailsWithoutSecretValue(t *testing.T) {
|
||||
env := NewEnvironment(map[string]string{"PRESENT": "do-not-print"}, emptyLookup)
|
||||
_, err := env.ResolveCredentials(Credentials{AccessKeyIDEnv: "MISSING"})
|
||||
if err == nil {
|
||||
t.Fatal("ResolveCredentials() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "MISSING") {
|
||||
t.Fatalf("ResolveCredentials() error = %q, want missing variable name", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "do-not-print") {
|
||||
t.Fatalf("ResolveCredentials() error exposed secret value: %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCredentialsRejectsEmptyReferencedValue(t *testing.T) {
|
||||
env := NewEnvironment(map[string]string{"EMPTY": ""}, emptyLookup)
|
||||
_, err := env.ResolveCredentials(Credentials{AccessKeyIDEnv: "EMPTY"})
|
||||
if err == nil {
|
||||
t.Fatal("ResolveCredentials() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "EMPTY") || !strings.Contains(err.Error(), "empty") {
|
||||
t.Fatalf("ResolveCredentials() error = %q, want empty variable context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeSecret(t *testing.T, directory, name, value string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(directory, name), []byte(value), 0o600); err != nil {
|
||||
t.Fatalf("write secret %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertEnvValue(t *testing.T, env Environment, name, want string) {
|
||||
t.Helper()
|
||||
got, ok := env.Lookup(name)
|
||||
if !ok {
|
||||
t.Fatalf("Lookup(%q) ok = false", name)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("Lookup(%q) = %q, want %q", name, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func emptyLookup(string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
func mapLookup(values map[string]string) EnvLookup {
|
||||
return func(name string) (string, bool) {
|
||||
value, ok := values[name]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
64
internal/config/ssh.go
Normal file
64
internal/config/ssh.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type HostKeyPolicy string
|
||||
|
||||
const (
|
||||
HostKeyPolicyStrict HostKeyPolicy = "strict"
|
||||
HostKeyPolicyAcceptNew HostKeyPolicy = "accept-new"
|
||||
HostKeyPolicyOff HostKeyPolicy = "off"
|
||||
)
|
||||
|
||||
func (p *HostKeyPolicy) UnmarshalYAML(value *yaml.Node) error {
|
||||
switch value.Kind {
|
||||
case yaml.ScalarNode:
|
||||
default:
|
||||
return fmt.Errorf("host_key_policy must be a boolean or string")
|
||||
}
|
||||
|
||||
switch value.Tag {
|
||||
case "!!bool":
|
||||
var enabled bool
|
||||
if err := value.Decode(&enabled); err != nil {
|
||||
return err
|
||||
}
|
||||
if enabled {
|
||||
*p = HostKeyPolicyStrict
|
||||
} else {
|
||||
*p = HostKeyPolicyOff
|
||||
}
|
||||
return nil
|
||||
case "!!str":
|
||||
var raw string
|
||||
if err := value.Decode(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
normalized, ok := NormalizeHostKeyPolicy(raw)
|
||||
if !ok {
|
||||
return fmt.Errorf("host_key_policy must be strict, true, accept-new, off, or false")
|
||||
}
|
||||
*p = normalized
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("host_key_policy must be a boolean or string")
|
||||
}
|
||||
}
|
||||
|
||||
func NormalizeHostKeyPolicy(value string) (HostKeyPolicy, bool) {
|
||||
switch strings.ToLower(value) {
|
||||
case "", string(HostKeyPolicyAcceptNew):
|
||||
return HostKeyPolicyAcceptNew, true
|
||||
case string(HostKeyPolicyStrict), "true":
|
||||
return HostKeyPolicyStrict, true
|
||||
case string(HostKeyPolicyOff), "false":
|
||||
return HostKeyPolicyOff, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ func Validate(cfg Config) error {
|
||||
pipelineIDs[pipeline.ID] = struct{}{}
|
||||
}
|
||||
|
||||
errs = validateBackend(errs, pipelineContext+".source", pipeline.Source.Backend, pipeline.Source.Path, pipeline.Source.URI, pipeline.Source.Endpoint, pipeline.Source.Bucket)
|
||||
errs = validateSourceBackend(errs, pipelineContext+".source", pipeline.Source)
|
||||
errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation)
|
||||
if len(pipeline.Destinations) == 0 {
|
||||
errs = append(errs, pipelineContext+".destinations is required")
|
||||
@@ -56,7 +56,7 @@ func Validate(cfg Config) error {
|
||||
destinationIDs[destination.ID] = struct{}{}
|
||||
}
|
||||
|
||||
errs = validateBackend(errs, destinationContext, destination.Backend, destination.Path, destination.URI, destination.Endpoint, destination.Bucket)
|
||||
errs = validateDestinationBackend(errs, destinationContext, destination)
|
||||
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
|
||||
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
|
||||
}
|
||||
@@ -68,7 +68,15 @@ func Validate(cfg Config) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoint, bucket string) ValidationErrors {
|
||||
func validateSourceBackend(errs ValidationErrors, context string, backend Backend) ValidationErrors {
|
||||
return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.Endpoint, backend.Bucket, backend.Prefix, backend.SSH.HostKeyPolicy, backend.Creds)
|
||||
}
|
||||
|
||||
func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors {
|
||||
return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.Endpoint, destination.Bucket, destination.Prefix, destination.SSH.HostKeyPolicy, destination.Creds)
|
||||
}
|
||||
|
||||
func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, endpoint, bucket, prefix string, hostKeyPolicy HostKeyPolicy, creds Credentials) ValidationErrors {
|
||||
switch backend {
|
||||
case "":
|
||||
errs = append(errs, context+".backend is required")
|
||||
@@ -77,12 +85,23 @@ func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoin
|
||||
errs = append(errs, context+".path is required for local backend")
|
||||
}
|
||||
case BackendSSH:
|
||||
if uri == "" {
|
||||
errs = append(errs, context+".uri is required for ssh backend")
|
||||
if host == "" {
|
||||
errs = append(errs, context+".host is required for ssh backend")
|
||||
}
|
||||
if path == "" {
|
||||
errs = append(errs, context+".path is required for ssh backend")
|
||||
}
|
||||
if port < 0 || port > 65535 {
|
||||
errs = append(errs, context+".port must be between 1 and 65535")
|
||||
}
|
||||
if port == 0 {
|
||||
errs = append(errs, context+".port is required for ssh backend after defaults are applied")
|
||||
}
|
||||
if hostKeyPolicy != "" {
|
||||
if _, ok := NormalizeHostKeyPolicy(string(hostKeyPolicy)); !ok {
|
||||
errs = append(errs, context+".host_key_policy must be strict, true, accept-new, off, or false")
|
||||
}
|
||||
}
|
||||
case BackendS3:
|
||||
if endpoint == "" {
|
||||
errs = append(errs, context+".endpoint is required for s3 backend")
|
||||
@@ -90,6 +109,12 @@ func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoin
|
||||
if bucket == "" {
|
||||
errs = append(errs, context+".bucket is required for s3 backend")
|
||||
}
|
||||
if err := ValidateS3Prefix(prefix); err != nil {
|
||||
errs = append(errs, context+".prefix must be a clean relative slash-separated path")
|
||||
}
|
||||
if (creds.AccessKeyIDEnv == "") != (creds.SecretAccessKeyEnv == "") {
|
||||
errs = append(errs, context+".credentials.access_key_id_env and credentials.secret_access_key_env must be configured together")
|
||||
}
|
||||
default:
|
||||
errs = append(errs, context+".backend "+backend+" is unsupported")
|
||||
}
|
||||
@@ -146,11 +171,11 @@ func validateTransferPolicy(errs ValidationErrors, context string, policy Transf
|
||||
if policy.OnDestinationOlder != TransferActionReplace && policy.OnDestinationOlder != TransferActionFail {
|
||||
errs = append(errs, context+".on_destination_older must be replace or fail")
|
||||
}
|
||||
if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail {
|
||||
errs = append(errs, context+".on_destination_newer must be skip or fail")
|
||||
if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail && policy.OnDestinationNewer != TransferActionReplace {
|
||||
errs = append(errs, context+".on_destination_newer must be skip, replace, or fail")
|
||||
}
|
||||
if policy.OnConflict != TransferActionFail {
|
||||
errs = append(errs, context+".on_conflict must be fail")
|
||||
if policy.OnConflict != TransferActionFail && policy.OnConflict != TransferActionReplace {
|
||||
errs = append(errs, context+".on_conflict must be fail or replace")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
@@ -47,6 +47,29 @@ func TestValidateChecksPublishTransformPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAcceptsForceReplacementTransferActions(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{
|
||||
Backend: BackendLocal,
|
||||
Path: "/source",
|
||||
},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Transfer: TransferPolicy{
|
||||
OnDestinationNewer: TransferActionReplace,
|
||||
OnConflict: TransferActionReplace,
|
||||
},
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type publishTransformPolicyCase struct {
|
||||
name string
|
||||
publish PublishPolicy
|
||||
|
||||
@@ -14,7 +14,7 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
switch plan.Action {
|
||||
case ActionSkipSame, ActionSkipDestinationNewer:
|
||||
return nil
|
||||
case ActionPublishNew, ActionReplaceOlder:
|
||||
case ActionPublishNew, ActionReplaceOlder, ActionForceReplace:
|
||||
default:
|
||||
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
||||
}
|
||||
@@ -30,6 +30,14 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if plan.Action == ActionForceReplace {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
cleanup := func() {
|
||||
|
||||
241
internal/publish/force_test.go
Normal file
241
internal/publish/force_test.go
Normal file
@@ -0,0 +1,241 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prepare func(t *testing.T, backend *fake.Backend, source bundle.Manifest)
|
||||
transfer config.TransferPolicy
|
||||
wantReason string
|
||||
forceAction bool
|
||||
}{
|
||||
{
|
||||
name: "unmanaged content",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
writeFakeFile(t, backend, "bundle/old.txt", "old")
|
||||
},
|
||||
transfer: defaultTransfer(),
|
||||
wantReason: "fail_unmanaged",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "different source id",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
conflict := source
|
||||
conflict.ID = "other.source"
|
||||
writeFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
},
|
||||
transfer: conflictReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "same created digest conflict",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
conflict := testutil.ValidManifest(testutil.BundleOptions{Files: []testutil.SourceFile{{Path: "report.md", Data: "# Different\n"}}})
|
||||
writeFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
},
|
||||
transfer: conflictReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "pipeline mismatch",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
writeFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{PipelineID: "other-pipeline"})
|
||||
},
|
||||
transfer: conflictReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "destination mismatch",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
writeFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{DestinationID: "other-destination"})
|
||||
},
|
||||
transfer: conflictReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "newer destination",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
newer := source
|
||||
newer.Created = newer.Created.AddDate(0, 0, 1)
|
||||
writeFakeDestinationState(t, backend, "bundle", newer, testutil.DestinationStateOptions{})
|
||||
},
|
||||
transfer: newerReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
tt.prepare(t, destinationBackend, sourceBundle.Manifest)
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, tt.transfer)
|
||||
_, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantReason) {
|
||||
t.Fatalf("Build() error = %v, want %q", err, tt.wantReason)
|
||||
}
|
||||
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if tt.forceAction {
|
||||
if err != nil {
|
||||
t.Fatalf("Build() with force error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace || !plan.Force {
|
||||
t.Fatalf("forced plan action = %s force=%t", plan.Action, plan.Force)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequiresConflictPolicyForStateConflicts(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
conflict := sourceBundle.Manifest
|
||||
conflict.ID = "other.source"
|
||||
writeFakeDestinationState(t, destinationBackend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
||||
req.Force = true
|
||||
_, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "destination source id differs") {
|
||||
t.Fatalf("Build() error = %v, want conservative conflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteForcedReplacementDeletesOnlyBundlePath(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeFile(t, destinationBackend, "bundle/old.txt", "old")
|
||||
writeFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
|
||||
writeFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
writeFakeFile(t, destinationBackend, "outside.txt", "outside")
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace {
|
||||
t.Fatalf("plan action = %s, want force_replace", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
assertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
||||
assertFakeMissing(t, destinationBackend, "bundle/old.txt")
|
||||
assertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
|
||||
assertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
assertFakeFile(t, destinationBackend, "outside.txt", "outside")
|
||||
}
|
||||
|
||||
func forceRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, transfer config.TransferPolicy) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Transfer: transfer,
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func defaultTransfer() config.TransferPolicy {
|
||||
return config.TransferPolicy{
|
||||
OnDestinationSame: config.TransferActionSkip,
|
||||
OnDestinationOlder: config.TransferActionReplace,
|
||||
OnDestinationNewer: config.TransferActionSkip,
|
||||
OnConflict: config.TransferActionFail,
|
||||
}
|
||||
}
|
||||
|
||||
func conflictReplaceTransfer() config.TransferPolicy {
|
||||
transfer := defaultTransfer()
|
||||
transfer.OnConflict = config.TransferActionReplace
|
||||
return transfer
|
||||
}
|
||||
|
||||
func newerReplaceTransfer() config.TransferPolicy {
|
||||
transfer := defaultTransfer()
|
||||
transfer.OnDestinationNewer = config.TransferActionReplace
|
||||
return transfer
|
||||
}
|
||||
|
||||
func writeFakeDestinationState(t *testing.T, backend *fake.Backend, relative string, manifest bundle.Manifest, opts testutil.DestinationStateOptions) {
|
||||
t.Helper()
|
||||
destinationState := testutil.DestinationState(manifest, opts)
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal destination state: %v", err)
|
||||
}
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
writeFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||
for _, output := range destinationState.Outputs {
|
||||
path, err := storage.Join(relative, output.Path)
|
||||
if err != nil {
|
||||
t.Fatalf("join output path: %v", err)
|
||||
}
|
||||
writeFakeFile(t, backend, path, "old")
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeFile(t *testing.T, backend *fake.Backend, path, data string) {
|
||||
t.Helper()
|
||||
if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
|
||||
t.Fatalf("write fake file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFakeFile(t *testing.T, backend *fake.Backend, path, want string) {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("read fake file %s: %v", path, err)
|
||||
}
|
||||
if got := string(data); got != want {
|
||||
t.Fatalf("fake file %s = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFakeMissing(t *testing.T, backend *fake.Backend, path string) {
|
||||
t.Helper()
|
||||
if _, err := backend.Stat(context.Background(), path); !storage.IsNotFound(err) {
|
||||
t.Fatalf("fake file %s stat error = %v, want not found", path, err)
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ const (
|
||||
ActionSkipDestinationNewer Action = "skip_destination_newer"
|
||||
ActionFailConflict Action = "fail_conflict"
|
||||
ActionFailUnmanaged Action = "fail_unmanaged"
|
||||
ActionForceReplace Action = "force_replace"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
@@ -34,6 +35,7 @@ type Request struct {
|
||||
Transformers TransformerResolver
|
||||
Transfer config.TransferPolicy
|
||||
DistributorVersion string
|
||||
Force bool
|
||||
}
|
||||
|
||||
type TransformerResolver interface {
|
||||
@@ -48,6 +50,7 @@ type Plan struct {
|
||||
DestinationBundlePath string
|
||||
Action Action
|
||||
Reason string
|
||||
Force bool
|
||||
Outputs []Output
|
||||
ExistingState *state.DistributorState
|
||||
}
|
||||
@@ -75,7 +78,7 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
return Plan{}, err
|
||||
}
|
||||
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
|
||||
action, reason := actionForComparison(comparison, req.Transfer)
|
||||
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
|
||||
plan := Plan{
|
||||
PipelineID: req.PipelineID,
|
||||
DestinationID: req.DestinationID,
|
||||
@@ -84,6 +87,7 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
DestinationBundlePath: req.DestinationBundlePath,
|
||||
Action: action,
|
||||
Reason: reason,
|
||||
Force: action == ActionForceReplace,
|
||||
Outputs: outputs,
|
||||
ExistingState: status.State,
|
||||
}
|
||||
@@ -112,13 +116,24 @@ func validateRequest(req Request) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy) (Action, string) {
|
||||
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
|
||||
switch comparison.Outcome {
|
||||
case state.OutcomeDestinationAbsent:
|
||||
return ActionPublishNew, comparison.Reason
|
||||
case state.OutcomeDestinationUnmanaged:
|
||||
if force {
|
||||
return ActionForceReplace, "forced replacement of unmanaged destination content"
|
||||
}
|
||||
return ActionFailUnmanaged, comparison.Reason
|
||||
case state.OutcomeInvalidState, state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
|
||||
case state.OutcomeInvalidState:
|
||||
return ActionFailConflict, comparison.Reason
|
||||
case state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
|
||||
if transfer.OnConflict == config.TransferActionReplace {
|
||||
if force {
|
||||
return ActionForceReplace, "forced replacement of conflicting destination state: " + comparison.Reason
|
||||
}
|
||||
return ActionFailConflict, "destination conflict replacement requires --force"
|
||||
}
|
||||
return ActionFailConflict, comparison.Reason
|
||||
case state.OutcomeSameSource:
|
||||
if transfer.OnDestinationSame == config.TransferActionFail {
|
||||
@@ -131,6 +146,12 @@ func actionForComparison(comparison state.Comparison, transfer config.TransferPo
|
||||
}
|
||||
return ActionReplaceOlder, comparison.Reason
|
||||
case state.OutcomeDestinationNewer:
|
||||
if transfer.OnDestinationNewer == config.TransferActionReplace {
|
||||
if force {
|
||||
return ActionForceReplace, "forced replacement of newer destination state"
|
||||
}
|
||||
return ActionFailConflict, "destination is newer and replacement requires --force"
|
||||
}
|
||||
if transfer.OnDestinationNewer == config.TransferActionFail {
|
||||
return ActionFailConflict, "destination is newer and transfer policy requires failure"
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ type Backend interface {
|
||||
Walk(ctx context.Context, prefix string, opts WalkOptions, fn WalkFunc) error
|
||||
HasAny(ctx context.Context, prefix string) (bool, error)
|
||||
DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts DeleteOptions) error
|
||||
DeletePrefix(ctx context.Context, prefix string, opts DeleteOptions) error
|
||||
}
|
||||
|
||||
type WalkOptions struct {
|
||||
|
||||
@@ -29,6 +29,7 @@ const (
|
||||
OpWalk = "walk"
|
||||
OpHasAny = "has any"
|
||||
OpDeleteManagedBundle = "delete managed bundle"
|
||||
OpDeletePrefix = "delete prefix"
|
||||
OpRegisterBackend = "register backend"
|
||||
OpOpenBackend = "open backend"
|
||||
)
|
||||
|
||||
@@ -205,6 +205,41 @@ func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, ma
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) DeletePrefix(ctx context.Context, prefix string, opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := storage.ValidatePrefix(prefix); err != nil {
|
||||
return err
|
||||
}
|
||||
if prefix != "" && !b.exists(prefix) && !b.hasChild(prefix) {
|
||||
if opts.IgnoreMissing {
|
||||
return nil
|
||||
}
|
||||
return storage.NewError(storage.OpDeletePrefix, backendName, prefix, storage.ErrNotFound, nil)
|
||||
}
|
||||
for path := range b.files {
|
||||
if path == prefix || entryBelow(prefix, path) {
|
||||
delete(b.files, path)
|
||||
}
|
||||
}
|
||||
for path := range b.symlinks {
|
||||
if path == prefix || entryBelow(prefix, path) {
|
||||
delete(b.symlinks, path)
|
||||
}
|
||||
}
|
||||
for path := range b.dirs {
|
||||
if path != "" && (path == prefix || entryBelow(prefix, path)) {
|
||||
delete(b.dirs, path)
|
||||
}
|
||||
}
|
||||
if opts.PruneEmptyDirs {
|
||||
b.pruneEmptyParents(parentOf(prefix))
|
||||
}
|
||||
b.dirs[""] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) ensureParents(path string) {
|
||||
parent := parentOf(path)
|
||||
for parent != "" {
|
||||
|
||||
@@ -140,6 +140,30 @@ func TestBackendManagedDeletion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendDeletePrefixStaysWithinPrefix(t *testing.T) {
|
||||
backend := New()
|
||||
mustWrite(t, backend, "bundle/report.md", "report")
|
||||
mustWrite(t, backend, "bundle/nested/old.txt", "old")
|
||||
mustWrite(t, backend, "bundle-sibling/keep.txt", "keep")
|
||||
mustWrite(t, backend, "outside.txt", "outside")
|
||||
|
||||
if err := backend.DeletePrefix(context.Background(), "bundle", storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
t.Fatalf("DeletePrefix() error = %v", err)
|
||||
}
|
||||
if _, err := backend.Stat(context.Background(), "bundle/report.md"); !storage.IsNotFound(err) {
|
||||
t.Fatalf("deleted file stat error = %v, want not found", err)
|
||||
}
|
||||
if _, err := backend.Stat(context.Background(), "bundle/nested/old.txt"); !storage.IsNotFound(err) {
|
||||
t.Fatalf("deleted nested file stat error = %v, want not found", err)
|
||||
}
|
||||
if _, err := backend.Stat(context.Background(), "bundle-sibling/keep.txt"); err != nil {
|
||||
t.Fatalf("sibling stat error = %v", err)
|
||||
}
|
||||
if _, err := backend.Stat(context.Background(), "outside.txt"); err != nil {
|
||||
t.Fatalf("outside stat error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendHasAnyAndWalkStop(t *testing.T) {
|
||||
backend := New()
|
||||
found, err := backend.HasAny(context.Background(), "missing")
|
||||
|
||||
@@ -206,3 +206,7 @@ func (b walkBackend) HasAny(context.Context, string) (bool, error) {
|
||||
func (b walkBackend) DeleteManagedBundle(context.Context, string, []string, DeleteOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b walkBackend) DeletePrefix(context.Context, string, DeleteOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user