Consolidate future roadmap documentation

.woodpecker/release.yml

@@ -0,0 +1,50 @@
+when:
+  - event: tag
+
+steps:
+  - name: build-release-assets
+    image: golang:1.26
+    commands:
+      - |
+        set -eu
+
+        version="$CI_COMMIT_TAG"
+        dist="dist"
+        pkg="gitea.maximumdirect.net/eric/distributor/cmd/distributor"
+
+        rm -rf "$dist"
+        mkdir -p "$dist"
+
+        build_binary() {
+          goos="$1"
+          goarch="$2"
+          suffix="$3"
+          output="$dist/distributor-$version-$goos-$goarch$suffix"
+
+          CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
+            go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/distributor/internal/buildinfo.Version=$version" \
+            -o "$output" "$pkg"
+        }
+
+        build_binary linux amd64 ""
+        build_binary linux arm64 ""
+        build_binary darwin amd64 ""
+        build_binary darwin arm64 ""
+        build_binary windows amd64 ".exe"
+        build_binary windows arm64 ".exe"
+
+  - name: publish-release
+    image: woodpeckerci/plugin-release
+    depends_on:
+      - build-release-assets
+    settings:
+      api_key:
+        from_secret: GITEA_RELEASE_TOKEN
+      files:
+        - dist/distributor-*
+      checksum: sha256
+      checksum-file: SHA256SUMS
+      checksum-flatten: true
+      file-exists: skip
+      overwrite: false
+      prerelease: false

README.md

@@ -1,13 +1,20 @@
 # distributor
 
-`distributor` validates manifested report bundles and publishes selected source or generated artifacts to configured destinations.
+`distributor` validates manifested report bundles, plans destination updates, and publishes selected source files or generated HTML outputs to configured destinations.
 
-It is currently a local-first CLI: source bundles are read from local storage, destinations are local directories, and Markdown files can be rendered to HTML sidecars.
+It is a local-first Go CLI for report distribution. A pipeline reads one source bundle tree, validates `manifest.json`, fans out to one or more local, SSH/SFTP, or S3-compatible destinations, records destination state in `.distributor.json`, and can also accept authenticated tar or tar.gz uploads through the HTTP upload server.
 
-Run the local example pipeline:
+Run the maintained local example:
 
 ```sh
 go run ./cmd/distributor run --config examples/local-publish.yml
 ```
 
-See [docs/cli.md](docs/cli.md), [docs/config.md](docs/config.md), [docs/operations.md](docs/operations.md), and [docs/troubleshooting.md](docs/troubleshooting.md) for the implemented CLI, configuration, operating notes, and common failure modes. Planning material lives under `docs/roadmap/`.
+Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build, write, parse, and validate local source bundles with the same manifest contract used by the CLI. They can use `gitea.maximumdirect.net/eric/distributor/pkg/upload` to build or validate a bundle and submit it to `distributor serve` with bearer authentication and idempotency keys. See [Source bundle contract](docs/integrations/source-bundle.md) and [HTTP upload contract](docs/integrations/http-upload.md).
+
+- [CLI reference](docs/cli.md)
+- [Configuration reference](docs/config.md)
+- [Operations guide](docs/operations.md)
+- [Troubleshooting](docs/troubleshooting.md)
+- [Integration contracts](docs/integrations/source-bundle.md)
+- [Development architecture](docs/policy/architecture.md)

docs/cli.md

@@ -1,99 +1,222 @@
 # Distributor CLI
 
-## Shortest useful command
+Audience: operators, integrators, and developers who run `distributor` from a shell or automation.
+
+This document is the canonical command and flag reference. Configuration schema details live in [Configuration](config.md), operational recovery guidance lives in [Operations](operations.md), failure diagnosis lives in [Troubleshooting](troubleshooting.md), and external contracts live under [Integrations](integrations/source-bundle.md).
+
+## Shortest Useful Command
+
+Run the maintained local publishing example from the repository root:
 
 ```sh
 go run ./cmd/distributor run --config examples/local-publish.yml
 ```
 
-This discovers the example source bundle and publishes source files to `workspace/published/source-bundle`.
+The example reads `examples/source-bundle/manifest.json`, publishes the configured files into `workspace/published/source-bundle`, and writes destination state metadata beside the published output.
 
-## Command overview
+## Command Overview
 
-```sh
+```text
 distributor [--help]
-distributor version
-distributor run [--config <path>] [--dry-run]
-distributor validate <path>
-distributor inspect <path>
+distributor help
+distributor version [--format text|json]
+distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
+distributor serve [--config <path>]
+distributor validate [--format text|json] <path>
+distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
+distributor inspect [--format text|json] <path>
+distributor inspect --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
+distributor manifest
+distributor manifest create <bundle-path> --id <bundle-id> [options]
+distributor manifest create --id <bundle-id> [options] <bundle-path>
 ```
 
-- `version`: prints the application name and version. Development builds print `distributor dev`.
-- `run`: loads a YAML config, discovers local source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary.
-- `validate`: validates a local source bundle directory or a local tree containing source bundles.
-- `inspect`: validates local source bundles and prints normalized bundle metadata.
+- `version` prints the application name and version.
+- `run` executes configured pipelines against their destinations.
+- `serve` starts the authenticated HTTP upload API defined by the configuration file.
+- `validate` checks a local bundle path or a configured source bundle.
+- `inspect` reports manifest and file metadata for a local bundle path or a configured source bundle.
+- `manifest create` writes a `manifest.json` file for an existing bundle directory.
 
-`validate` and `inspect` accept local paths only. `run` currently executes local backends only. SSH and S3 config can be parsed and validated, but configured SSH or S3 execution fails with a clear unsupported-execution error.
+## Flag Reference
 
-## Flag reference
+### Help
 
-Root command:
+`distributor`, `distributor --help`, `distributor -h`, `distributor help`, and `distributor manifest` print command help. Unknown commands and invalid argument combinations print usage guidance and exit non-zero.
 
-- `--help`, `-h`, or `help`: print root help.
+### Common Output Format
 
-All subcommands:
+`--format text|json` is supported by `version`, `run`, `validate`, `inspect`, and `manifest create`.
 
-- `--help`, `-h`: print command-specific help.
+- `text` is the default human-readable output.
+- `json` emits one JSON document for successful command execution.
+- Invalid formats are rejected before command execution.
 
-`run` flags:
+### `version`
 
-- `--config <path>`: config file to load. If omitted, `run` uses `/usr/local/etc/distributor/config.yml`.
-- `--dry-run`: load config, discover bundles, inspect destination state, print planned actions and final status, and do not write files.
+```sh
+distributor version [--format text|json]
+```
 
-`run` does not accept positional arguments. `validate` and `inspect` accept at most one path; omitting the path returns a required-path error.
+`version` accepts no positional arguments. Text output prints the application name and version; JSON output includes `application` and `version` fields.
 
-## Common workflows
+### `run`
 
-Validate a source bundle:
+```sh
+distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
+```
+
+- `--config <path>` loads the pipeline configuration. If omitted, the application uses `/usr/local/etc/distributor/config.yml`.
+- `--dry-run` validates inputs and reports destination actions without applying changes.
+- `--force` permits a run when destination state indicates a conservative safety check would otherwise block it.
+- `--format text|json` selects human-readable or machine-readable output.
+
+`run` accepts no positional arguments.
+
+### `serve`
+
+```sh
+distributor serve [--config <path>]
+```
+
+- `--config <path>` loads HTTP, source, destination, and pipeline configuration. If omitted, the application uses `/usr/local/etc/distributor/config.yml`.
+
+`serve` accepts no positional arguments and runs until interrupted or until the server exits with an error.
+
+### `validate`
+
+```sh
+distributor validate [--format text|json] <path>
+distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
+```
+
+`validate` has two source modes:
+
+- Local path mode validates the bundle at `<path>`.
+- Configured source mode resolves the source from `--config <path>` and `--pipeline <id>`.
+
+Configured source flags:
+
+- `--config <path>` loads the configuration file.
+- `--pipeline <id>` selects the configured pipeline source to validate.
+- `--bundle <path>` overrides the configured source bundle path for the selected pipeline.
+- `--format text|json` selects output format.
+
+A local positional path cannot be combined with `--config`, `--pipeline`, or `--bundle`. When any configured source flag is used, both `--config` and `--pipeline` are required.
+
+### `inspect`
+
+```sh
+distributor inspect [--format text|json] <path>
+distributor inspect --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
+```
+
+`inspect` uses the same source mode rules as `validate`, then reports bundle metadata instead of only validation status. Local path mode requires exactly one bundle path. Configured source mode requires both `--config` and `--pipeline`; `--bundle` may override the selected pipeline source path.
+
+### `manifest create`
+
+```sh
+distributor manifest create <bundle-path> --id <bundle-id> [options]
+distributor manifest create --id <bundle-id> [options] <bundle-path>
+```
+
+Flags may appear before or after the bundle path. Both `--flag value` and `--flag=value` forms are accepted.
+
+- `--id <bundle-id>` sets the manifest bundle identifier and is required.
+- `--created <timestamp>` sets the manifest creation timestamp. If omitted, the current UTC time is used.
+- `--file <relative-path>` includes one file in the manifest. The flag may be repeated.
+- `--overwrite` allows replacing an existing `manifest.json` file.
+- `--format text|json` selects output format.
+
+If no `--file` flags are provided, `manifest create` scans the bundle directory recursively. The command requires exactly one bundle path, refuses unsafe manifest paths, and writes `manifest.json` at the bundle root.
+
+## Common Workflows
+
+### Validate Or Inspect A Local Bundle
 
 ```sh
 go run ./cmd/distributor validate examples/source-bundle
+go run ./cmd/distributor inspect --format json examples/source-bundle
 ```
 
-Inspect a source bundle:
+### Validate Or Inspect A Configured Source
 
 ```sh
-go run ./cmd/distributor inspect examples/source-bundle
+go run ./cmd/distributor validate --config examples/local-publish.yml --pipeline example-source-bundle
+go run ./cmd/distributor inspect --config examples/local-publish.yml --pipeline example-source-bundle --format json
 ```
 
-Preview local publication without writing:
+Use `--bundle <path>` with configured source mode when automation needs to validate or inspect an alternate bundle path through the selected pipeline configuration.
+
+### Create A Manifest
+
+```sh
+go run ./cmd/distributor manifest create examples/source-bundle --id example-source-bundle --overwrite
+go run ./cmd/distributor manifest create --id example-source-bundle --overwrite examples/source-bundle
+```
+
+Use repeated `--file` flags when the manifest should include an explicit file list instead of the recursive directory scan:
+
+```sh
+go run ./cmd/distributor manifest create examples/source-bundle \
+  --id example-source-bundle \
+  --file report.md \
+  --file summary.txt \
+  --overwrite
+```
+
+### Preview Or Publish A Pipeline
 
 ```sh
 go run ./cmd/distributor run --config examples/local-publish.yml --dry-run
-```
-
-Publish the local source example:
-
-```sh
 go run ./cmd/distributor run --config examples/local-publish.yml
 ```
 
-Publish the local HTML example:
-
-```sh
-go run ./cmd/distributor run --config examples/local-html.yml
-```
-
-Preview local fan-out publication:
+Use `--format json` when automation needs structured run results. Use `--force` only when the operator has reviewed the destination state conflict and intentionally wants to continue.
+
+### Run HTML And Fan-Out Examples
 
 ```sh
+go run ./cmd/distributor run --config examples/local-html.yml --dry-run
+go run ./cmd/distributor run --config examples/local-index.yml --dry-run
 go run ./cmd/distributor run --config examples/fan-out.yml --dry-run
+go run ./cmd/distributor run --config examples/archive-and-latest.yml --dry-run
 ```
 
-## Output
+These examples exercise implemented output rendering and destination planning behavior. They still use the same `run` flags and output contract described here.
 
-`run` prints the number of configured pipelines, one line per pipeline, one line per planned destination action, and a final status line. Actions include:
+### Start The HTTP Upload Server
 
-- `publish_new`: destination has no managed state and is empty.
-- `replace_older`: destination state is older than the source manifest.
-- `skip_same`: destination state already matches the source manifest.
-- `skip_destination_newer`: destination state is newer than the source manifest.
-- `error`: planning or execution failed for that destination.
+```sh
+go run ./cmd/distributor serve --config examples/http-upload-local.yml
+```
 
-The command exits non-zero if any destination fails. Independent later destinations are still attempted.
+The server exposes health, status, and authenticated upload endpoints according to the loaded configuration. Use [Operations](operations.md) for server operation and recovery guidance.
 
-## Diagnostics
+## Output And Exit Behavior
 
-Use `validate` before publication when a producer has written a new bundle. Use `inspect` to confirm normalized ids, timestamps, digests, file paths, and file sizes.
+Text output is optimized for direct operator use. JSON output is optimized for automation and uses a command-specific result object with a shared envelope similar to:
 
-For symptom-oriented recovery steps, see [troubleshooting](troubleshooting.md). For destination state and retry behavior, see [operations](operations.md). For config fields and defaults, see [configuration](config.md).
+```json
+{
+  "schema_version": 1,
+  "command": "inspect",
+  "ok": true,
+  "warnings": [],
+  "result": {}
+}
+```
+
+- Successful JSON commands emit one JSON document on stdout.
+- Usage errors and fatal setup errors exit non-zero and do not emit a JSON result document.
+- `run --format json` emits a JSON result for partial destination failures, sets `ok` to `false`, includes result details and errors, and exits non-zero.
+- Warnings are included in JSON output and are printed in text output when relevant.
+
+## Diagnostics And Recovery
+
+- Use `validate` before `run` when checking a bundle supplied by another process.
+- Use `inspect --format json` when automation needs manifest metadata, normalized file details, or checksum information.
+- Use `run --dry-run` before publishing to review destination actions.
+- Use [Configuration](config.md) for schema and default details.
+- Use [Troubleshooting](troubleshooting.md) for common errors and corrective action.
+- Use [Operations](operations.md) for HTTP upload operation, state files, and recovery workflows.

docs/config.md

@@ -1,16 +1,25 @@
-# Distributor Configuration
+# Configuration Reference
 
-## Config File Location
+Audience: administrators, operators, and advanced users who write YAML configuration for `distributor`.
 
-`distributor run --config <path>` loads the YAML config at the provided path.
+This document is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operating procedures live in [Operations](operations.md), symptom-oriented recovery lives in [Troubleshooting](troubleshooting.md), and external contracts live under [Integrations](integrations/source-bundle.md).
 
-If `--config` is omitted, `run` uses:
+## Config File Loading
+
+`distributor run --config <path>` and `distributor serve --config <path>` load the YAML file at `<path>`. If `--config` is omitted, both commands use:
 
 ```text
 /usr/local/etc/distributor/config.yml
 ```
 
-Config parsing rejects unknown YAML fields. The current executable backend support is local only. SSH and S3 config fields are accepted by config validation, but runtime execution for those backends is unavailable.
+YAML decoding rejects unknown fields. Defaults are applied after decoding and before validation.
+
+Runtime backend support is command-specific:
+
+- `run`, `validate --config`, and `inspect --config` execute `local`, `ssh`, and `s3` sources.
+- `run` executes `local`, `ssh`, and `s3` destinations.
+- `serve` uses `http_upload` sources through the HTTP upload API and publishes to configured `local`, `ssh`, and `s3` destinations.
+- `http_upload` is valid only as a source backend.
 
 ## Minimal Local Config
 
@@ -26,11 +35,21 @@ pipelines:
         path: /srv/reports/archive
 ```
 
-This publishes source files only. It uses the default validation and transfer policies.
+This config publishes source files only. It uses default validation, destination path mapping, publish, transfer, and HTTP server values.
 
 ## Production-Oriented Local Config
 
 ```yaml
+server:
+  http:
+    bind: 127.0.0.1:8080
+    staging_root: /var/spool/distributor
+    max_upload_size: 20MB
+    queue_size: 16
+    max_concurrency: 1
+    retention: 24h
+secrets:
+  directory: /run/secrets/distributor
 pipelines:
   - id: reports
     source:
@@ -45,6 +64,8 @@ pipelines:
         publish:
           source: true
           html: false
+        path_mapping:
+          mode: preserve_relative
         transfer:
           on_destination_same: skip
           on_destination_older: replace
@@ -52,9 +73,214 @@ pipelines:
           on_conflict: fail
 ```
 
-## HTML Publication
+## HTTP Upload Source Config
 
-To publish generated HTML from Markdown files:
+HTTP upload sources are configured on pipelines and are served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML.
+
+```yaml
+server:
+  http:
+    bind: 127.0.0.1:8080
+    staging_root: /var/spool/distributor
+    max_upload_size: 20MB
+    queue_size: 16
+    max_concurrency: 1
+    retention: 24h
+pipelines:
+  - id: weather-daily
+    source:
+      backend: http_upload
+      token_env: WEATHER_DAILY_UPLOAD_TOKEN
+      staging_path: /var/spool/distributor/weather-daily
+      max_upload_size: 20MB
+    destinations:
+      - id: archive
+        backend: local
+        path: /srv/reports/archive
+```
+
+`token_env` is required for `http_upload` sources. `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`. `max_upload_size` defaults to `server.http.max_upload_size`.
+
+`serve` maps each resolved bearer token to exactly one `http_upload` pipeline. Startup fails when a token is missing, empty, or duplicates another upload pipeline token.
+
+## Top-Level Fields
+
+### `server.http`
+
+`server.http` controls the HTTP upload server used by `serve`.
+
+- `bind`: optional TCP bind address. Default: `127.0.0.1:8080`.
+- `staging_root`: optional root used to default `http_upload` source staging paths. Default: `/var/spool/distributor`.
+- `max_upload_size`: optional default upload limit for HTTP upload sources. Default: `20MB`.
+- `queue_size`: optional upload admission queue size. Default: `16`.
+- `max_concurrency`: optional upload worker concurrency. Default: `1`.
+- `retention`: optional in-memory completed-run retention duration. Default: `24h`.
+
+Numeric server values and durations must be greater than zero after defaults are applied.
+
+### `secrets`
+
+- `directory`: optional directory of secret files used by the config-owned credential resolver.
+
+See [Secrets](#secrets) for resolution rules.
+
+### `pipelines`
+
+`pipelines` is required and must contain at least one pipeline.
+
+Each pipeline has:
+
+- `id`: required unique slug-like identifier. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
+- `source`: required source backend config.
+- `validation`: optional validation policy.
+- `destinations`: required non-empty destination list.
+
+Pipeline ids must be unique across the config.
+
+## Backend Reference
+
+### Local Backend
+
+Local backends can be used as sources and destinations.
+
+```yaml
+backend: local
+path: /srv/distributor/archive
+```
+
+- `backend`: required value `local`.
+- `path`: required local filesystem root for this backend.
+
+### SSH/SFTP Backend
+
+SSH backends use native SFTP and can be used as sources and destinations. Adapter protocol behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md).
+
+```yaml
+backend: ssh
+host: ssh.example.com
+user: distributor
+port: 22
+path: /srv/distributor/archive
+ssh_key_file: /home/distributor/.ssh/id_ed25519
+known_hosts: /home/distributor/.ssh/known_hosts
+host_key_policy: strict
+```
+
+- `backend`: required value `ssh`.
+- `host`: required SSH host.
+- `path`: required remote root path.
+- `user`: optional SSH username. If omitted, the adapter uses the current OS user when available.
+- `port`: optional TCP port. Default: `22`.
+- `ssh_key_file`: optional private key path.
+- `known_hosts`: optional OpenSSH `known_hosts` path.
+- `host_key_policy`: optional host key policy. Default: `accept-new`.
+
+Accepted host key policy values:
+
+- `strict` or boolean `true`: require a matching known host key.
+- `accept-new`: accept and persist a new host key, but reject changed known keys.
+- `off` or boolean `false`: disable host key checking.
+
+Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML.
+
+### S3-Compatible Backend
+
+S3 backends can be used as sources and destinations. Adapter protocol behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md).
+
+```yaml
+backend: s3
+endpoint: https://s3.example.com
+bucket: reports
+prefix: distributor/archive
+region: us-east-1
+force_path_style: true
+credentials:
+  access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
+  secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
+```
+
+- `backend`: required value `s3`.
+- `endpoint`: required S3-compatible endpoint URL.
+- `bucket`: required bucket name.
+- `prefix`: optional backend root prefix. Leading and trailing slashes are trimmed; the remaining value must be a clean relative slash-separated path.
+- `region`: optional region. Default: `us-east-1`.
+- `force_path_style`: optional addressing mode toggle. Default: `true`.
+- `credentials.access_key_id_env`: optional environment variable or secret-file name for the access key id.
+- `credentials.secret_access_key_env`: optional environment variable or secret-file name for the secret access key.
+
+The S3 credential variable names must either both be configured or both be omitted. When omitted, the AWS SDK default credential chain is used. When configured, both values must resolve to non-empty strings through the process environment or `secrets.directory`.
+
+### HTTP Upload Source Backend
+
+HTTP upload backends are valid only as pipeline sources and are served by `distributor serve`. The API contract is documented in [HTTP Upload API Contract](integrations/http-upload.md).
+
+```yaml
+backend: http_upload
+token_env: WEATHER_DAILY_UPLOAD_TOKEN
+staging_path: /var/spool/distributor/weather-daily
+max_upload_size: 20MB
+```
+
+- `backend`: required value `http_upload`.
+- `token_env`: required environment variable or secret-file name containing the bearer token.
+- `staging_path`: optional staging path. Default: `<server.http.staging_root>/<pipeline id>`.
+- `max_upload_size`: optional per-source upload limit. Default: `server.http.max_upload_size`.
+
+## Validation Policy
+
+```yaml
+validation:
+  on_digest_mismatch: fail
+```
+
+- `validation.on_digest_mismatch`: optional. Default and only accepted value: `fail`.
+
+Source bundle digest mismatches fail validation before destination writes occur. The manifest file-format contract is documented in [Source Bundle Contract](integrations/source-bundle.md).
+
+## Destination Fields
+
+Each destination embeds a backend config at the destination level and may also configure publishing, transforms, path mapping, links, and transfer behavior.
+
+```yaml
+destinations:
+  - id: archive
+    backend: local
+    path: /srv/reports/archive
+    publish:
+      source: true
+      html: false
+    path_mapping:
+      mode: preserve_relative
+    transfer:
+      on_destination_same: skip
+      on_destination_older: replace
+      on_destination_newer: skip
+      on_conflict: fail
+```
+
+- `id`: required unique slug-like identifier within the pipeline.
+- Backend fields: required according to the selected destination backend.
+- `publish`: optional publish policy. Default: source-only publication.
+- `transform`: required only when publishing generated HTML.
+- `path_mapping`: optional destination path mapping policy.
+- `links`: optional public URL metadata policy.
+- `transfer`: optional destination reconciliation policy.
+
+Destination ids must be unique within a pipeline.
+
+## Publish And Transform Policy
+
+### Source-Only Publication
+
+```yaml
+publish:
+  source: true
+  html: false
+```
+
+`publish.source` controls whether source manifest files are copied to the destination.
+
+### Markdown-To-HTML Publication
 
 ```yaml
 publish:
@@ -66,69 +292,108 @@ transform:
     mode: sidecar
 ```
 
-Sidecar generation writes `report.html` for `report.md`. It does not mutate the source bundle.
+`publish.html` controls whether generated HTML outputs are published. When `publish.html` is `true`, `transform.markdown_to_html.enabled` must also be `true`.
 
-## Reference
+Markdown transform fields:
 
-Top level:
+- `transform.markdown_to_html.enabled`: enables Markdown-to-HTML generation for this destination.
+- `transform.markdown_to_html.mode`: optional. Accepted values are `sidecar` and `index`; default is `sidecar` when a Markdown transform block is present.
+- `transform.markdown_to_html.input`: optional source manifest path for `index` mode only.
 
-- `pipelines`: required non-empty list.
+`sidecar` mode renders every manifest-listed `.md` file to a same-directory `.html` output. `index` mode renders one Markdown source to `index.html` at the destination bundle path. If `index` mode omits `input`, the selected source bundle must contain exactly one Markdown file.
 
-Pipeline:
+At least one output type must be enabled. Enabled Markdown transforms are rejected when `publish.html` is `false`, and `input` is rejected unless `mode` is `index`.
 
-- `id`: required unique slug-like identifier.
-- `source`: required backend config.
-- `validation.on_digest_mismatch`: optional; defaults to `fail`; only `fail` is supported.
-- `destinations`: required non-empty destination list.
+## Destination Path Mapping
 
-Source backend:
+```yaml
+path_mapping:
+  mode: preserve_relative
+```
 
-- `backend`: required.
-- `path`: required for `local` and `ssh`.
-- `uri`: required for `ssh`.
-- `endpoint`: required for `s3`.
-- `bucket`: required for `s3`.
-- `prefix`: optional for `s3`.
-- `region`: optional for `s3`.
-- `force_path_style`: optional for `s3`.
-- `credentials.access_key_id_env`: optional S3 credential environment variable name.
-- `credentials.secret_access_key_env`: optional S3 credential environment variable name.
+- `path_mapping.mode`: optional. Accepted values are `preserve_relative` and `fixed`; default is `preserve_relative`.
 
-Destination:
+`preserve_relative` publishes each discovered source bundle at the same path relative to the destination backend root.
 
-- `id`: required unique slug-like identifier within the pipeline.
-- Backend fields: same accepted shape as source backends, with destination fields at the destination level.
-- `publish`: optional; defaults to source-only publication.
-- `transform`: required only for generated HTML publication.
-- `transfer`: optional; defaults described below.
+`fixed` publishes one selected source bundle directly at the destination backend root. Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple bundles have the same timestamp, the source-root-relative bundle path in ascending order wins.
 
-Accepted backend names:
+Fixed mapping is useful for stable latest-style paths. Preview fixed destinations with `run --dry-run`, especially before using `--force`.
 
-- `local`: executable; requires `path`.
-- `ssh`: config validation only; execution is unavailable.
-- `s3`: config validation only; execution is unavailable.
+## Destination Links
 
-Publish policy:
+```yaml
+links:
+  base_url: https://reports.example.com/archive
+  primary: auto
+```
 
-- `publish.source`: publish source artifacts.
-- `publish.html`: publish generated HTML artifacts from Markdown source files.
+- `links.base_url`: required when `links` is present. It must be an absolute `http` or `https` URL with a host and no query string or fragment.
+- `links.primary`: optional. Accepted values are `auto`, `html`, and `source`; default is `auto` when `links` is present.
 
-At least one output type must be enabled. When `publish.html` is true, `transform.markdown_to_html.enabled` must be `true` and `transform.markdown_to_html.mode` must be `sidecar`.
+`distributor` does not infer public URLs from backend config. Destination state URL fields are documented in [Destination State Contract](integrations/destination-state.md). Output URLs are built from `links.base_url`, the destination bundle path, and output paths using URL path semantics. `index.html` outputs produce directory-style URLs that omit the filename.
 
-Transfer policy:
+Primary URL policies:
 
-- `transfer.on_destination_same`: `skip` or `fail`; defaults to `skip`.
-- `transfer.on_destination_older`: `replace` or `fail`; defaults to `replace`.
-- `transfer.on_destination_newer`: `skip` or `fail`; defaults to `skip`.
-- `transfer.on_conflict`: only `fail`; defaults to `fail`.
+- `auto`: prefer `index.html`, then generated HTML, then source outputs.
+- `html`: use the first generated HTML output.
+- `source`: use the first copied source output.
+
+If no output matches the primary policy, per-output URLs may still be recorded and the top-level primary URL is omitted.
+
+## Transfer Policy
+
+```yaml
+transfer:
+  on_destination_same: skip
+  on_destination_older: replace
+  on_destination_newer: skip
+  on_conflict: fail
+```
+
+Transfer fields and accepted values:
+
+- `transfer.on_destination_same`: `skip` or `fail`. Default: `skip`.
+- `transfer.on_destination_older`: `replace` or `fail`. Default: `replace`.
+- `transfer.on_destination_newer`: `skip`, `replace`, or `fail`. Default: `skip`.
+- `transfer.on_conflict`: `fail` or `replace`. Default: `fail`.
+
+`replace` for `on_destination_newer` and `on_conflict` is honored only when `run --force` is supplied. There is no config field that enables forced replacement by default.
+
+## Size And Duration Values
+
+Upload size fields must be YAML strings with an integer and one of these suffixes:
+
+- `B`
+- `KB`
+- `MB`
+- `GB`
+
+Suffix multipliers use powers of 1024. Values must be greater than zero after defaults are applied.
+
+Duration fields must be YAML strings accepted by Go duration parsing, such as `24h`, `90m`, or `168h`. Values must be greater than zero after defaults are applied.
 
 ## Defaults
 
 Defaults are applied after YAML decoding and before validation:
 
+- `server.http.bind: 127.0.0.1:8080`
+- `server.http.staging_root: /var/spool/distributor`
+- `server.http.max_upload_size: 20MB`
+- `server.http.queue_size: 16`
+- `server.http.max_concurrency: 1`
+- `server.http.retention: 24h`
 - `validation.on_digest_mismatch: fail`
-- `publish.source: true`
-- `publish.html: false`
+- SSH `port: 22`
+- SSH `host_key_policy: accept-new`
+- S3 `region: us-east-1`
+- S3 `prefix`: leading and trailing slashes trimmed
+- S3 `force_path_style: true`
+- `http_upload` source `staging_path: <server.http.staging_root>/<pipeline id>`
+- `http_upload` source `max_upload_size: server.http.max_upload_size`
+- `publish.source: true` and `publish.html: false`
+- `transform.markdown_to_html.mode: sidecar` when a Markdown transform block is present and mode is omitted
+- `path_mapping.mode: preserve_relative`
+- `links.primary: auto` when a `links` block is present and `primary` is omitted
 - `transfer.on_destination_same: skip`
 - `transfer.on_destination_older: replace`
 - `transfer.on_destination_newer: skip`
@@ -136,18 +401,38 @@ Defaults are applied after YAML decoding and before validation:
 
 ## Secrets
 
-Do not put literal secrets in config files. S3 credentials may name environment variables:
+Do not put literal secrets in config files. `secrets.directory` lets deployments provide credential values as files:
+
+```yaml
+secrets:
+  directory: /run/secrets/distributor
+```
+
+Each regular file in the directory becomes an internal credential environment value named by the filename. Valid filenames match `[A-Za-z_][A-Za-z0-9_]*`. Directories are ignored. Symlinks to regular files are followed. Exactly one trailing LF or CRLF is trimmed from each file; other whitespace is preserved.
+
+Credential resolution checks the real process environment first, then `secrets.directory`. If both define the same name with different values, `run` emits a warning with the variable name and uses the real environment value. Secret values are not printed. The process environment is not modified, so SDK default credential chains see only real process environment variables.
+
+Fields resolved through this resolver:
 
 - `credentials.access_key_id_env`
 - `credentials.secret_access_key_env`
+- `source.token_env` for `http_upload` sources
 
-S3 execution is unavailable; these fields are accepted so config shape can be validated.
+## Maintained Examples
 
-## Examples
+Maintained examples live under [examples](../examples/). Config tests load every file listed here.
 
-Maintained examples live under [examples](../examples/):
+Local examples:
 
-- `local-to-local.yml`: minimal local config.
-- `local-publish.yml`: runnable local source publication.
-- `local-html.yml`: runnable local HTML publication.
-- `fan-out.yml`: runnable local fan-out publication to source and HTML destinations.
+- `local-to-local.yml`: minimal local-to-local config using absolute sample paths; load-tested, but paths should be adapted before running.
+- `local-publish.yml`: runnable local source publication used by the README quickstart.
+- `local-html.yml`: local sidecar HTML publication.
+- `local-index.yml`: local `index.html` publication.
+- `fan-out.yml`: local fan-out publication to source and HTML destinations.
+- `archive-and-latest.yml`: local archive plus fixed latest publication.
+- `http-upload-local.yml`: local HTTP upload server config; requires `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` in the process environment or as a secret-file name before running `serve`.
+
+Environment-gated remote examples:
+
+- `ssh-destination.yml`: local-to-SSH publication; replace host, user, path, key, and known-host values for an SSH/SFTP endpoint you control.
+- `s3-destination.yml`: local-to-S3 publication; replace endpoint, bucket, prefix, region, and credential variable names for an S3-compatible service you control.

docs/integrations/destination-state.md

@@ -0,0 +1,99 @@
+# Destination State Contract
+
+Audience: operators, integrators, and maintainers who inspect or reason about destination `.distributor.json` files.
+
+Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for comparison, skip, replacement, and recovery decisions.
+
+## State Schema
+
+Current schema version: `1`.
+
+```json
+{
+  "schema_version": 1,
+  "distributor_version": "dev",
+  "pipeline_id": "reports",
+  "destination_id": "archive",
+  "published_at": "2026-06-04T12:00:00Z",
+  "source": {
+    "manifest": {
+      "schema_version": 1,
+      "id": "reports.example.2026-06-04",
+      "digest": "sha256:...",
+      "created": "2026-06-04T11:55:00Z",
+      "files": [
+        {"path": "report.md", "sha256": "sha256:...", "size": 1234}
+      ]
+    }
+  },
+  "links": {
+    "primary_url": "https://reports.example.com/archive/report.html"
+  },
+  "outputs": [
+    {
+      "path": "report.html",
+      "kind": "generated",
+      "source_path": "report.md",
+      "transform": "markdown_to_html",
+      "url": "https://reports.example.com/archive/report.html",
+      "sha256": "sha256:...",
+      "size": 2345
+    }
+  ]
+}
+```
+
+Required fields:
+
+- `schema_version`: must be `1`.
+- `pipeline_id`: configured pipeline id that wrote the state.
+- `destination_id`: configured destination id that wrote the state.
+- `published_at`: RFC3339 publication timestamp.
+- `source.manifest`: embedded source bundle manifest.
+- `outputs`: output records array; empty is allowed, but the field is required.
+
+Optional fields:
+
+- `distributor_version`: application version string when available.
+- `links.primary_url`: absolute HTTP or HTTPS URL selected by destination link policy.
+
+## Output Records
+
+Each output record has:
+
+- `path`: destination-relative output path.
+- `kind`: `source` or `generated`.
+- `source_path`: source manifest path used for the output.
+- `transform`: required for `generated` outputs; omitted for copied source outputs.
+- `url`: optional absolute HTTP or HTTPS URL for the output.
+- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes.
+- `size`: output byte size, zero or greater.
+
+Output paths must be unique and use clean relative slash-separated path rules.
+
+## Comparison Semantics
+
+`distributor` compares the current source manifest to destination state before writing:
+
+- No state and no content: publish new outputs.
+- No state and existing content: treat the destination as unmanaged.
+- Matching embedded source manifest: skip.
+- Same source id with older `created`: replace if policy allows.
+- Same source id with newer `created`: skip by default.
+- Same source id and same `created` with different digest: conflict.
+- Different source id, pipeline id, or destination id: conflict.
+- Invalid state JSON or invalid state fields: conflict.
+
+Normal replacement deletes only managed output paths recorded in `outputs` plus `.distributor.json`. Forced replacement deletes the bounded destination bundle path.
+
+## Boundaries
+
+Destination state is internal managed state written by `distributor`. Operators may inspect it during recovery, but normal workflows should not edit it by hand. Source `manifest.json` is not copied as destination state.
+
+## Tests
+
+Before changing this contract, inspect and run:
+
+```sh
+go test ./internal/state ./internal/publish
+```

docs/integrations/http-upload.md

@@ -0,0 +1,149 @@
+# HTTP Upload API Contract
+
+Audience: producers, operators, and maintainers integrating with `distributor serve`.
+
+`distributor serve` exposes a local HTTP upload API for pipelines whose source backend is `http_upload`. Each bearer token maps to exactly one configured pipeline.
+
+## Authentication
+
+Uploads authenticate with:
+
+```text
+Authorization: Bearer <token>
+```
+
+Token values are resolved from the configured `source.token_env` through the process environment or `secrets.directory`. Tokens are not configured as YAML literal values.
+
+Requests that include `pipeline` or `pipeline_id` query parameters are rejected. The bearer token selects the pipeline.
+
+## Endpoints
+
+### `GET /healthz`
+
+Returns `200 OK` when the server is running:
+
+```json
+{"status":"ok"}
+```
+
+### `POST /upload`
+
+Accepts one source bundle archive and returns after the archive is staged and validated.
+
+Producers may include:
+
+```text
+Idempotency-Key: <key>
+```
+
+Idempotency keys are scoped to the authenticated pipeline selected by the bearer token. Valid keys are non-empty ASCII strings up to 128 bytes using letters, digits, `.`, `_`, `-`, and `:`. Invalid keys return `400`.
+
+Accepted content types:
+
+- `application/x-tar`
+- `application/gzip`
+- `application/x-gzip`
+
+Successful admission returns `202 Accepted`:
+
+```json
+{"run_id":"reports.20260604T120000Z.abcdef12","status":"accepted"}
+```
+
+Common error responses:
+
+- `400`: pipeline query supplied, invalid idempotency key, archive rejected, malformed archive, or invalid staged source bundle.
+- `401`: missing, empty, or unknown bearer token.
+- `409`: repeated idempotency key conflicts with another source manifest, or the same key is already being staged.
+- `413`: upload body exceeds the selected pipeline size limit.
+- `415`: unsupported content type.
+- `503`: upload queue is full.
+
+Error bodies use:
+
+```json
+{"error":"<message>"}
+```
+
+Retryable idempotency conflicts include:
+
+```json
+{"error":"upload idempotency key is already being processed","retryable":true}
+```
+
+When `Idempotency-Key` is omitted, upload admission preserves the raw HTTP behavior: every valid accepted upload receives its own run id. When a key is supplied, the server records the accepted run after archive staging and source bundle validation succeed. Reusing the same key for the same authenticated pipeline and the same normalized source manifest returns the original `202 Accepted` response and does not enqueue another run. Reusing the same key for a different normalized source manifest returns `409 Conflict`.
+
+### `GET /runs/<run-id>`
+
+Returns an in-memory status record while retained:
+
+```json
+{
+  "run_id": "reports.20260604T120000Z.abcdef12",
+  "pipeline_id": "reports",
+  "status": "succeeded",
+  "accepted_at": "2026-06-04T12:00:00Z",
+  "started_at": "2026-06-04T12:00:01Z",
+  "finished_at": "2026-06-04T12:00:02Z",
+  "report": {}
+}
+```
+
+Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Failed records include `error`. Succeeded and failed records may include a run report.
+
+Unknown, malformed, expired, or process-lost run ids return `404`.
+
+## Archive Contract
+
+Upload archives must be uncompressed tar or gzip-compressed tar. The archive must contain exactly one root-level `manifest.json` and all manifest-listed files.
+
+Archive entry rules:
+
+- Paths must be clean relative slash-separated paths.
+- Absolute paths, backslashes, `.` and `..` segments, duplicate files, and nested `manifest.json` entries are rejected.
+- Only directories and regular files are accepted.
+- Symlinks, hardlinks, devices, FIFOs, sockets, and other entry types are rejected.
+
+The uploaded archive size and extracted bundle size are bounded by the selected pipeline's `source.max_upload_size`. Extracted file count is also bounded by the implementation.
+
+## Go Producer Helper
+
+Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/upload` to build or validate source bundles, package them as gzip-compressed tar archives, and submit them to this API:
+
+```go
+client, err := upload.NewClient(upload.ClientOptions{
+	Endpoint: "http://127.0.0.1:8080",
+	Token:    token,
+})
+if err != nil {
+	return err
+}
+result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
+	Root:           "examples/source-bundle",
+	IdempotencyKey: "reports.example.20260604T120000Z",
+})
+```
+
+`Endpoint` is the server base URL; the package derives `/upload` and `/runs/<run-id>`. `UploadBundle` validates a local bundle by default and uploads only `manifest.json` plus manifest-listed files. `UploadFiles` creates a temporary bundle from explicit `bundle.BundleFile` values before uploading. When `IdempotencyKey` is omitted, the package generates one random 128-bit lowercase hex key for the upload operation and reuses it across retries.
+
+The helper retries only safe cases: `503 Service Unavailable`, temporary network errors, and ambiguous mid-upload failures. It does not retry after `202 Accepted` and does not retry `400`, `401`, `409`, `413`, or `415`. Bearer token values are redacted from returned errors.
+
+## Queue And Retention
+
+`server.http.queue_size` bounds accepted-but-not-started uploads plus uploads being staged. `server.http.max_concurrency` bounds publishing concurrency. The coordinator does not run two uploads for the same pipeline concurrently.
+
+Completed status records expire after `server.http.retention`; expiration removes committed staged bundle directories for completed uploads. Server restart clears queue state and status records.
+
+Idempotency records are memory-only, expire with completed upload status records, and are cleared by server restart.
+
+## Boundaries
+
+The HTTP API does not expose pipeline selection by request parameter, TLS, public routing policy, or durable status storage. Put public access controls, TLS termination, and rate limiting in deployment infrastructure.
+
+## Tests
+
+Before changing this contract, inspect and run:
+
+```sh
+go test ./internal/app ./internal/ingest ./pkg/upload
+```

docs/integrations/markdown.md

@@ -1,22 +1,27 @@
 # Markdown Integration
 
-## Purpose
+Audience: operators and maintainers who rely on generated HTML outputs from Markdown source files.
 
-Markdown-to-HTML is the only implemented external file-format integration. This note documents the renderer behavior that is externally visible in generated destination artifacts.
+Markdown-to-HTML is an implemented file-format integration used by destination transform policy.
 
 ## Dependency
 
-Rendering uses `github.com/yuin/goldmark`. The exact dependency version is pinned in `go.mod`; review that file before changing renderer behavior or diagnosing version-specific output changes.
+Rendering uses `github.com/yuin/goldmark`. The exact version is pinned in `go.mod`.
 
-## Renderer behavior
+## Renderer Behavior
 
-`internal/transform/markdown.New` constructs the renderer with `goldmark.New()` and no project-specific extensions or renderer options.
+The transformer constructs `goldmark.New()` with no project-specific extensions, parser options, renderer options, templates, CSS, or metadata injection.
 
-For each source bundle file ending in `.md`, the transform reads the Markdown source and generates an HTML sidecar in the same logical directory. The output path replaces the `.md` suffix with `.html`, so `report.md` produces `report.html`. Non-Markdown source files produce no Markdown outputs.
+Supported output modes:
+
+- `sidecar`: renders each source manifest file ending in `.md` and writes a generated output beside it with the `.md` suffix replaced by `.html`.
+- `index`: renders one Markdown source file to `index.html` at the destination bundle path.
+
+In `index` mode, `transform.markdown_to_html.input` may name the source manifest path to render. If `input` is omitted, the source manifest must list exactly one `.md` file. The selected input must be a clean relative source path, must be listed in the source manifest, and must end in `.md`.
 
 Raw HTML embedded in Markdown is not passed through by the current renderer behavior. Tests allow Goldmark's disabled-or-escaped raw HTML output forms and reject literal script tags in generated HTML.
 
-## Wrapper
+## HTML Wrapper
 
 Rendered Markdown body HTML is wrapped in a fixed document shell:
 
@@ -28,7 +33,7 @@ Rendered Markdown body HTML is wrapped in a fixed document shell:
 
 The wrapper is deterministic and does not read configuration, templates, CSS, or source manifest metadata.
 
-## Output metadata
+## Output Metadata
 
 Generated outputs record:
 
@@ -38,18 +43,16 @@ Generated outputs record:
 - SHA-256 digest of the wrapped HTML bytes;
 - byte size of the wrapped HTML bytes.
 
+Destination state stores generated outputs with `kind: generated`, `source_path`, `transform`, `sha256`, `size`, and optional `url`.
+
 ## Boundaries
 
-Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select outputs, or choose transfer actions. Publish planning decides whether generated HTML is selected for a destination.
-
-Only sidecar output mode is supported for current behavior.
+Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose transfer policy. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata.
 
 ## Tests
 
 Before changing Markdown renderer behavior, inspect and run:
 
-```bash
+```sh
 go test ./internal/transform/markdown
 ```
-
-The tests cover sidecar naming, ignored non-Markdown files, raw HTML handling, deterministic output, digest metadata, and size metadata.

docs/integrations/s3.md

@@ -0,0 +1,73 @@
+# S3-Compatible Storage Integration
+
+Audience: operators and maintainers configuring S3-compatible sources or destinations.
+
+The S3 backend uses the AWS SDK for Go v2 against a configured S3-compatible endpoint.
+
+## Dependencies
+
+Runtime S3 support uses:
+
+- `github.com/aws/aws-sdk-go-v2`
+- `github.com/aws/aws-sdk-go-v2/config`
+- `github.com/aws/aws-sdk-go-v2/credentials`
+- `github.com/aws/aws-sdk-go-v2/service/s3`
+- `github.com/aws/smithy-go`
+
+Exact versions are pinned in `go.mod`.
+
+## Config Contract
+
+Required fields:
+
+- `backend: s3`
+- `endpoint`
+- `bucket`
+
+Optional fields:
+
+- `prefix`: backend root object-key prefix; leading and trailing slashes are trimmed.
+- `region`: defaults to `us-east-1`.
+- `force_path_style`: defaults to `true` through config defaults.
+- `credentials.access_key_id_env`
+- `credentials.secret_access_key_env`
+
+Credential environment variable names must either both be configured or both be omitted. When configured, values are resolved through the process environment or `secrets.directory` before opening the backend. When omitted, the AWS SDK default credential chain is used.
+
+## Object Key Mapping
+
+The configured `bucket` plus optional `prefix` is the backend root. Logical storage paths are joined under that prefix using slash-separated object keys.
+
+Prefixes and logical paths must be clean relative slash-separated paths. Prefixes are object-key prefixes, not real directories.
+
+## Storage Behavior
+
+The adapter uses these S3 operations:
+
+- `HeadObject` for stat and overwrite checks.
+- `GetObject` for reads.
+- `PutObject` for writes.
+- `ListObjectsV2` for walks and prefix deletion planning.
+- `DeleteObject` for managed cleanup and replacement.
+
+Writes buffer the input and set `ContentLength`. If no content type is supplied by the caller, the adapter infers a content type from the logical path.
+
+Normal replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Forced replacement deletes objects under the bounded destination bundle prefix. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
+
+## Error Mapping
+
+S3 not-found responses are translated into shared storage not-found errors where practical. Other service, credential, network, or endpoint errors are surfaced through storage errors with backend and logical path context.
+
+## Boundaries
+
+The S3 integration does not create buckets, alter bucket policy, configure TLS bypass, manage public URLs, or infer website URLs from endpoint or bucket settings. Public URL metadata is configured separately with destination `links`.
+
+## Tests
+
+Before changing this integration, inspect and run:
+
+```sh
+go test ./internal/adapters/s3
+```
+
+Live S3-compatible tests are opt-in and gated by environment variables in the adapter test package.

docs/integrations/source-bundle.md

@@ -0,0 +1,94 @@
+# Source Bundle Contract
+
+Audience: producer developers, integrators, and maintainers who create or validate source bundles consumed by `distributor`.
+
+A source bundle is a directory containing `manifest.json` and every regular file listed by that manifest. This is the producer-to-`distributor` file-format contract.
+
+## Manifest Schema
+
+Current schema version: `1`.
+
+```json
+{
+  "schema_version": 1,
+  "id": "reports.example.2026-06-04",
+  "digest": "sha256:...",
+  "created": "2026-06-04T12:00:00Z",
+  "files": [
+    {
+      "path": "report.md",
+      "sha256": "sha256:...",
+      "size": 1234
+    }
+  ]
+}
+```
+
+Required manifest fields:
+
+- `schema_version`: must be `1`.
+- `id`: non-empty bundle identifier.
+- `digest`: lowercase `sha256:<64 hex>` digest of the ordered `files` list.
+- `created`: RFC3339 timestamp.
+- `files`: non-empty ordered list of file records.
+
+Required file fields:
+
+- `path`: bundle-relative slash-separated file path.
+- `sha256`: lowercase `sha256:<64 hex>` digest of the file bytes.
+- `size`: file size in bytes, zero or greater.
+
+## Path Rules
+
+Manifest file paths must be clean relative slash-separated paths. They must not be empty, absolute, contain backslashes, contain `.` or `..` segments, include empty path segments, or normalize to a different path.
+
+Any basename of `manifest.json` or `.distributor.json` is reserved, including nested occurrences such as `nested/manifest.json`.
+
+Listed files must be regular files. Symlinks and other special file types are rejected during local bundle validation and manifest building.
+
+## Digest Rules
+
+File digests use SHA-256 over each file's raw bytes.
+
+The bundle digest is SHA-256 over the canonical JSON-like payload for the ordered file records. The payload is constructed as:
+
+```text
+[{"path":"<path>","sha256":"<sha256>","size":<size>},...]
+```
+
+File order is significant. Explicit file lists preserve caller order. Scan mode sorts paths in ascending slash-path order.
+
+## Producer APIs
+
+Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build and validate this contract:
+
+- `BuildManifest`: builds a manifest from explicit file paths or scan mode.
+- `WriteManifest`: writes `manifest.json`, optionally replacing an existing manifest.
+- `WriteBundle`: copies source files into a complete bundle, validates it, and promotes it into place.
+- `LoadManifest`, `ParseManifest`, `ValidateManifest`, and `ValidateBundle`: parse and validate local bundles.
+- `FileDigest`, `BundleDigest`, and `ValidateDigest`: digest helpers.
+
+Go producers that submit bundles to `distributor serve` can use `gitea.maximumdirect.net/eric/distributor/pkg/upload`. It builds on `pkg/bundle`, packages valid bundles as gzip-compressed tar uploads, sends bearer authentication, and includes idempotency keys for safe retry behavior. See [HTTP Upload API Contract](http-upload.md).
+
+CLI producers can use:
+
+```sh
+go run ./cmd/distributor manifest create <bundle-path> --id <bundle-id>
+go run ./cmd/distributor validate <bundle-path>
+```
+
+## Scan Mode
+
+Manifest scan mode walks the local bundle root recursively, includes regular files, includes dotfiles, skips files whose basename is `manifest.json` or `.distributor.json`, rejects symlinks, and sorts paths before building the manifest.
+
+## Boundaries
+
+The source bundle manifest does not configure routing, destination selection, public URLs, credentials, transforms, notification behavior, or storage backends. Those concerns belong in `distributor` configuration and destination state.
+
+## Tests
+
+Before changing this contract, inspect and run:
+
+```sh
+go test ./pkg/bundle ./pkg/upload ./internal/bundle
+```

docs/integrations/ssh-sftp.md

@@ -0,0 +1,71 @@
+# SSH/SFTP Integration
+
+Audience: operators and maintainers configuring SSH/SFTP sources or destinations.
+
+The SSH backend uses native SSH and SFTP libraries. It does not call `ssh`, `scp`, or `rsync`.
+
+## Dependencies
+
+Runtime SSH support uses:
+
+- `golang.org/x/crypto/ssh`
+- `golang.org/x/crypto/ssh/agent`
+- `golang.org/x/crypto/ssh/knownhosts`
+- `github.com/pkg/sftp`
+
+Exact versions are pinned in `go.mod`.
+
+## Config Contract
+
+Required fields:
+
+- `backend: ssh`
+- `host`
+- `path`
+
+Optional fields:
+
+- `user`: defaults to the current OS user when available.
+- `port`: defaults to `22`.
+- `ssh_key_file`: private key path.
+- `known_hosts`: OpenSSH known-hosts file path.
+- `host_key_policy`: `strict`, `accept-new`, or `off`; defaults to `accept-new`.
+
+## Authentication
+
+Authentication methods are attempted in this order:
+
+1. SSH agent identities when `SSH_AUTH_SOCK` is set.
+2. The private key configured by `ssh_key_file`.
+
+Password authentication is not configured in YAML. If neither an agent nor key file is available, opening the backend fails.
+
+## Host Key Policy
+
+- `strict`: requires a matching known host key.
+- `accept-new`: accepts and persists an unknown host key, but rejects changed known keys.
+- `off`: disables host key checking.
+
+When `known_hosts` is omitted and checking is enabled, the adapter uses the current user's default OpenSSH `known_hosts` path when available. During dry runs, accepted unknown host keys are not persisted.
+
+## Storage Behavior
+
+The configured `path` is the backend root. All source discovery, destination paths, reads, writes, state files, and deletes operate on logical paths below that root.
+
+The adapter rejects symlink ancestors for reads and writes. Reads require regular files. Writes create parent directories and prefer atomic temp-file-plus-rename writes when requested. Walk output is sorted through the shared storage walker.
+
+Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Forced replacement deletes the bounded destination bundle path.
+
+## Boundaries
+
+The SSH backend does not configure passwords, jump hosts, shell commands, `rsync`, host-key bypass warnings beyond command output, or broad recursive deletion outside the destination bundle path.
+
+## Tests
+
+Before changing this integration, inspect and run:
+
+```sh
+go test ./internal/adapters/ssh
+```
+
+Live SSH tests are opt-in and gated by environment variables in the adapter test package.

docs/internal/app.md

@@ -1,64 +1,69 @@
 # Application Orchestration
 
+Audience: developers and LLM coding agents changing `internal/app`.
+
 ## Purpose
 
-`internal/app` owns top-level use cases for `run`, `validate`, and `inspect`. It wires configuration, storage backends, transforms, publish planning, execution, summaries, and notification handoff.
+`internal/app` owns top-level application use cases: run, single-pipeline run, staged-source run, validate, inspect, manifest creation, and HTTP upload serving. It coordinates config loading, secret resolution, backend construction, source discovery, destination selection, publish planning/execution, notification handoff, output projection, and upload coordination.
 
-## Inputs and outputs
+## Inputs And Outputs
 
-`Run` accepts a context, optional config path, dry-run flag, stdout writer, and optional notifier. It loads YAML config, discovers source bundles for each configured pipeline, plans each destination independently, optionally executes publish plans, writes summary output when stdout is supplied, and returns an aggregated error if any destination fails.
+Inputs include app option structs, contexts, config paths, pipeline ids, local source roots, dry-run/force flags, output format, stdout writers, HTTP requests, and optional notifier implementations.
 
-`Validate` and `Inspect` accept a local path. `Validate` discovers and validates bundles. `Inspect` writes bundle metadata and manifest file entries to stdout when provided.
-
-## Run flow
-
-The runner:
-
-1. loads config from the supplied path or `config.DefaultConfigPath`;
-2. opens the configured source backend;
-3. discovers validated bundles from the source root;
-4. opens each destination backend independently;
-5. builds a publish plan for each bundle and destination;
-6. prints plan lines and records summary counters;
-7. executes publish or replacement plans unless dry-run is enabled;
-8. invokes the notifier after successful publish or replacement actions.
-
-Destination failures are collected while later destinations continue to run. Source open and source discovery failures stop the run because there are no valid bundles to fan out.
-
-## Backend and transform wiring
-
-The app-level backend factory registers only the local backend for execution. Config validation accepts other backend shapes, but `Run` can execute only local sources and local destinations.
-
-The app-level transform registry registers Markdown-to-HTML using `internal/transform/markdown`. Lower-level publish code receives a resolver and does not import concrete transform implementations.
-
-## Dry-run behavior
-
-Dry-run still loads config, opens backends, discovers bundles, inspects destinations, resolves transforms, and builds publish plans. It does not write destination outputs, write `.distributor.json`, delete managed outputs, or notify.
-
-## Failure behavior
-
-`Run` returns immediately for config loading errors, context cancellation before work starts, source open errors, and source discovery errors. Per-destination backend, planning, execution, and notification errors are aggregated into one run error after remaining destinations have been attempted.
-
-Stdout write errors are returned immediately because the caller's requested output stream can no longer be trusted.
+Outputs include `RunReport`, validate/inspect/manifest results, CLI text/JSON projections, HTTP upload responses, upload status records, and errors. Destination-scoped failures can return a partial run report plus an aggregated error; fatal setup failures return before a complete report exists.
 
 ## Boundaries
 
-`internal/app` coordinates packages but does not own manifest validation rules, destination state comparison, storage path rules, output planning, transform rendering, or backend-specific filesystem behavior.
+`internal/app` wires packages together but does not own manifest validation rules, destination state comparison, storage path rules, publish safety policy, transform rendering, config schema validation, or backend protocol behavior.
 
-`Validate` and `Inspect` are local path commands. Remote execution wiring is outside current behavior.
+User-facing command parsing stays in `internal/cli`. User-facing config reference stays in `docs/config.md`. External contracts live under `docs/integrations/`.
 
-## Tests
+## Config Fields Used
 
-Before changing app orchestration, inspect tests under:
+The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, and transfer policy.
 
-- `internal/app`
-- `internal/cli`
-- `internal/publish`
+Config fields are validated and defaulted by `internal/config` before app workflows use them.
 
-## Invariants
+## Adapters Used
 
-- One source fans out to each destination independently.
-- Destination failures do not prevent later destinations from being planned.
-- Dry-run must not mutate destination storage or invoke notifications.
-- Concrete backend and transform registration stays at the app layer.
-- The default notifier is `notify.Noop`.
+The app backend factory registers runtime storage adapters for local filesystem, SSH/SFTP, and S3-compatible storage. It resolves explicit credentials through the config-owned environment resolver before opening S3 backends.
+
+The app layer registers default transforms, including Markdown-to-HTML, and supplies a transform resolver to publish planning. It uses `notify.Noop` when no notifier is supplied.
+
+## State And Manifest Behavior
+
+Run workflows discover and validate source bundles through `internal/bundle`. Destination state actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results.
+
+HTTP uploads stage and validate archives before enqueueing a pipeline run with a local staged source root. Go producers can use the public `pkg/upload` package to create client-side gzip tar uploads for this server contract; `internal/app` remains the server-side orchestration boundary and does not import that producer package.
+
+Upload idempotency is owned by the upload coordinator. Optional `Idempotency-Key` values are scoped to the authenticated pipeline. The coordinator reserves a key while staging is in progress, records the accepted run id with the validated source manifest identity after staging succeeds, returns the original accepted record for the same key and same manifest, and rejects the same key with a different manifest as a conflict.
+
+## Skip And Resume Behavior
+
+Fan-out destinations are independent. A destination failure is recorded and does not prevent later destinations from being attempted. Dry-run builds plans and reports without destination writes, destination state writes, notifier calls, or SSH known-host persistence.
+
+HTTP upload status is in memory. Accepted jobs move through accepted, queued, running, succeeded, or failed states and expire after configured retention. Upload idempotency records are also memory-only, expire with the completed status record for their accepted run, and are cleared by process restart.
+
+## Failure Behavior
+
+Runtime setup fails for config load, config validation, secret loading, or credential resolution errors. Source setup failures stop the affected run before destination planning. Destination open, planning, execution, and notification failures are recorded as destination failures where a partial result exists.
+
+HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Upload requests can fail during authentication, idempotency-key validation, content-type validation, idempotency conflict checks, queue admission, archive staging, source validation, or later publish execution.
+
+## Tests To Inspect
+
+- `internal/app/*_test.go`
+- `internal/cli/root_test.go`
+- `internal/config/*_test.go`
+- `internal/ingest/*_test.go`
+- `internal/publish/*_test.go`
+
+## Architectural Invariants
+
+- App orchestration owns wiring, not low-level policy.
+- Dry-run must not write outputs, destination state, notifier events, or SSH known-host entries.
+- Fan-out destinations remain independent after a destination-scoped failure.
+- Secret values are never printed; warnings may name variables only.
+- Upload admission stages and validates a bundle before returning a run id.
+- Idempotent upload retries compare normalized source manifest identity, not archive bytes.
+- Runtime backend registration remains app-owned.

docs/internal/bundle.md

@@ -1,53 +1,53 @@
-# Bundles
+# Source Bundle Internals
+
+Audience: developers and LLM coding agents changing `internal/bundle`.
 
 ## Purpose
 
-`internal/bundle` parses, discovers, and validates source bundles through the storage interface.
+`internal/bundle` discovers and validates source bundles through the storage abstraction. It adapts the public producer-facing source manifest contract from `pkg/bundle` to local, SSH/SFTP, S3-compatible, and test storage backends.
 
-## Inputs and outputs
+## Inputs And Outputs
 
-Input is a backend-rooted directory tree containing one or more `manifest.json` files. Output is a deterministic list of validated bundles with relative bundle paths and normalized manifest data.
-
-## Manifest behavior
-
-The source manifest requires:
-
-- `schema_version: 1`
-- `id`
-- `digest`
-- `created`
-- non-empty `files`
-
-Each file requires `path`, `sha256`, and `size`. Digests must use lowercase `sha256:<64 hex>` format. `created` must parse as RFC3339.
-
-## Validation
-
-`ValidateManifest` owns normalized source manifest semantics: schema version, id, digest format, timestamp presence, file list presence, source path safety, duplicate file paths, reserved paths, file digest format, non-negative file sizes, and the top-level bundle digest.
-
-Storage-backed bundle validation additionally checks file existence, regular-file type, file size, and per-file SHA-256.
-
-The bundle digest is SHA-256 of a deterministic JSON array of file records in manifest order with fields `path`, `sha256`, and `size`.
-
-## Discovery
-
-Discovery walks a storage backend beneath a source root, finds `manifest.json` files, sorts bundle paths lexically, and rejects nested manifests.
-
-## Failure behavior
-
-Manifest parsing and validation fail before destination planning. Storage-backed validation fails when listed files are missing, are not regular files, have unexpected sizes, have unexpected SHA-256 digests, or when a source bundle includes unsafe or reserved paths.
+Inputs are a context, a `storage.Backend`, and a source-root prefix or bundle root path. Outputs are sorted `Bundle` records containing the source-root-relative bundle path and validated manifest.
 
 ## Boundaries
 
-Bundle code uses `internal/storage` and does not import concrete adapters. CLI local path support is wired in `internal/app`.
+`internal/bundle` delegates manifest parsing, digest calculation, source path validation, and manifest validation to `pkg/bundle`. It does not publish files, inspect destination state, choose pipelines, or know concrete backend implementations.
 
-## Tests
+The external source bundle file-format contract is documented in `docs/integrations/source-bundle.md`.
 
-Before changing bundle behavior, inspect tests under `internal/bundle`.
+## Config Fields Used
 
-## Invariants
+The package does not read config directly. App workflows pass it storage backends that were opened from configured source fields.
 
-- `manifest.json` is the only source bundle contract.
-- Source file paths must stay relative to the bundle root.
-- The top-level bundle digest is derived from manifest file records in order.
-- Discovery order is lexical and deterministic.
-- Nested manifests are rejected.
+## Adapters Used
+
+The package depends only on `internal/storage.Backend`. Concrete local, SSH/SFTP, S3-compatible, and fake backends are hidden behind that interface.
+
+## State And Manifest Behavior
+
+Discovery walks recursively under the source root, finds entries whose basename is `manifest.json`, converts each manifest path to a bundle root, sorts roots, rejects nested bundle roots, and validates each bundle.
+
+Validation reads `manifest.json`, parses it, stats each manifest-listed file, requires regular files, verifies file sizes, reads file bytes, checks per-file SHA-256 digests, and recomputes the bundle digest.
+
+## Skip And Resume Behavior
+
+The package has no skip or resume state. Each call performs discovery or validation from the supplied backend state.
+
+## Failure Behavior
+
+Failures include invalid storage prefixes, missing manifests, parse errors, nested manifests, unsafe manifest paths, non-regular files, size mismatches, digest mismatches, backend stat/read errors, and no discovered bundles.
+
+## Tests To Inspect
+
+- `internal/bundle/*_test.go`
+- `pkg/bundle/*_test.go`
+- `internal/storage/fake/*_test.go`
+
+## Architectural Invariants
+
+- Source manifest semantics remain owned by `pkg/bundle`.
+- Discovery order is deterministic.
+- Nested manifests are rejected before returning bundles.
+- Source paths stay clean, relative, slash-separated, and confined to the backend root.
+- Concrete adapters never leak into bundle validation logic.

docs/internal/config.md

@@ -1,61 +1,57 @@
 # Configuration Internals
 
+Audience: developers and LLM coding agents changing `internal/config`.
+
 ## Purpose
 
-`internal/config` defines YAML-backed configuration structs, defaulting, and validation for distributor pipelines.
+`internal/config` owns YAML config structs, config file loading, defaulting, validation, size/duration parsing, SSH/S3 normalization helpers, and the credential environment resolver.
 
-## Inputs and outputs
+## Inputs And Outputs
 
-Input is a YAML file containing `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
+Inputs are YAML files, YAML scalar values, process environment lookup functions, optional secrets directories, and `Config` values. Outputs are defaulted `Config` values, validation errors, parsed byte sizes and durations, normalized backend options, loaded secret environments, secret conflict metadata, and resolved credentials.
 
-## Loading flow
+## Boundaries
 
-`LoadFile` opens the requested path, decodes YAML with known-field checking enabled, applies defaults, and validates the result. The app uses `DefaultConfigPath` when the CLI does not supply a config path.
+The package does not open storage backends, authenticate HTTP requests, start servers, publish destinations, or execute transforms. Runtime execution support is wired by `internal/app`.
 
-Known-field checking rejects misspelled or unknown YAML keys before defaults and validation run.
+The canonical user-facing config reference is `docs/config.md`.
 
-## Defaults
+## Config Fields Used
 
-Defaults are applied in `ApplyDefaults`:
+The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, and transfer policy.
 
-- pipeline validation defaults `on_digest_mismatch` to `fail`;
-- destination publish policy defaults to source output only;
-- `transfer.on_destination_same` defaults to `skip`;
-- `transfer.on_destination_older` defaults to `replace`;
-- `transfer.on_destination_newer` defaults to `skip`;
-- `transfer.on_conflict` defaults to `fail`.
+## Adapters Used
 
-## Validation responsibilities
+No external storage adapters are used directly. The package exposes normalized config and credential values consumed by app-level adapter construction.
 
-Validation requires at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, and valid transfer actions.
+## State And Manifest Behavior
 
-`ValidatePublishTransformPolicy` is shared with publish planning so destination policy combinations are checked consistently. Publishing HTML requires an enabled Markdown-to-HTML transform in `sidecar` mode. A publish policy must select source output, HTML output, or both.
+The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as publish/transform combinations, links, transfer policy, backend roots, S3 prefix shape, and HTTP upload source settings.
 
-## Executable support boundary
+## Skip And Resume Behavior
 
-Config validation accepts `local`, `ssh`, and `s3` backend shapes so config files can be validated as schemas. Runtime execution currently opens only local backends through `internal/app`.
+The package has no runtime skip or resume behavior. It provides transfer policy values that publish planning later applies to destination comparison outcomes.
 
-The user-facing configuration reference is `docs/config.md`; this file documents package behavior for maintainers.
+## Failure Behavior
 
-## Failure behavior
+`LoadFile` wraps file open, YAML parse, and validation failures with config path context. YAML decoding rejects unknown fields. Validation collects all detected field errors into a single error value.
 
-Load errors wrap the underlying file, YAML, or validation error with context. Validation collects all detected field errors into one error value instead of stopping at the first invalid field.
+Secret loading fails for unreadable secrets directories, invalid secret filenames, unreadable secret files, and missing or empty required credential values. Secret conflicts are returned as warnings metadata, not secret values.
 
-Unsupported backend names fail validation. Accepted backend names without runtime execution support fail later during app backend opening.
-
-## Tests
-
-Before changing config behavior, inspect:
+## Tests To Inspect
 
 - `internal/config/load_test.go`
 - `internal/config/validate_test.go`
-- example-loading coverage in `internal/config`
-- user-facing examples under `examples/`
+- `internal/config/secrets_test.go`
+- `internal/config/backend_view_test.go`
+- `internal/app/runtime_test.go`
+- example configs under `examples/`
 
-## Invariants
+## Architectural Invariants
 
 - Defaults are applied before validation.
 - Unknown YAML fields are rejected.
-- `docs/config.md` remains the canonical user-facing config reference.
-- Runtime backend execution support is not inferred from config validation support.
-- New user-visible config behavior must be covered by tests and docs in the same change.
+- `http_upload` is source-only config.
+- Credential-consuming runtime code must use the config-owned environment resolver.
+- Secret values are never printed by config warnings.
+- New user-visible config behavior must update `docs/config.md` and tests.

docs/internal/ingest.md

@@ -0,0 +1,51 @@
+# Ingestion Internals
+
+Audience: developers and LLM coding agents changing `internal/ingest`.
+
+## Purpose
+
+`internal/ingest` validates upload content types, extracts uploaded source bundle archives into local temporary storage, validates extracted bundles, and commits accepted bundles to per-run staging directories.
+
+## Inputs And Outputs
+
+Inputs are a context, upload body reader, content type, pipeline staging path, run id, maximum uploaded size, maximum extracted size, and maximum file count. Output is a `StagedBundle` containing the committed local bundle root and parsed manifest.
+
+## Boundaries
+
+The package does not authenticate HTTP requests, manage upload queues, track upload status, publish destinations, load config, or start an HTTP server. Those responsibilities live in `internal/app`.
+
+The HTTP API contract is documented in `docs/integrations/http-upload.md`.
+
+## Config Fields Used
+
+The package does not read config directly. The app layer passes effective values derived from `source.staging_path`, `source.max_upload_size`, and HTTP server defaults.
+
+## Adapters Used
+
+The package uses the local filesystem directly for temporary archive storage, extraction, validation, and final staging path promotion. It does not use the storage backend abstraction.
+
+## State And Manifest Behavior
+
+Accepted archives must contain exactly one root-level `manifest.json`. After extraction, the package validates the staged root through `pkg/bundle`, including manifest parsing, source path rules, file existence, regular-file checks, file sizes, file SHA-256 digests, and bundle digest.
+
+## Skip And Resume Behavior
+
+The package has no resume behavior. A successful call commits one complete staged bundle root. Failed calls remove temporary data created by that call.
+
+## Failure Behavior
+
+Failures include unsupported content type, unsafe run id, missing staging path, non-positive limits, oversize upload body, oversize extracted content, too many files, unsafe archive paths, duplicate files, nested manifests, unsupported tar entry types, gzip/tar read errors, bundle validation errors, and filesystem errors.
+
+## Tests To Inspect
+
+- `internal/ingest/archive_test.go`
+- `internal/app/upload_*_test.go`
+- `pkg/bundle/*_test.go`
+
+## Architectural Invariants
+
+- Invalid archives never commit a staged root.
+- Archive paths remain clean relative slash-separated paths.
+- Only directories and regular files are accepted from tar archives.
+- Source bundle validation happens before final staging path promotion.
+- Upload authentication and queueing remain outside this package.

docs/internal/link.md

@@ -0,0 +1,46 @@
+# Link URL Policy
+
+Audience: developers and LLM coding agents changing `internal/link`.
+
+## Purpose
+
+`internal/link` owns shared validation for configured and persisted HTTP link URLs.
+
+## Inputs And Outputs
+
+Input is a URL string. Output is either nil for an accepted URL or a concise validation error that callers wrap with field context.
+
+## Boundaries
+
+The package validates URL shape only. It does not construct output URLs, choose primary URLs, infer public URLs from backend configuration, parse config files, or write destination state.
+
+## Config Fields Used
+
+The package does not read config directly. `internal/config` uses it to validate `links.base_url`; `internal/state` uses it to validate persisted `links.primary_url` and output `url` fields.
+
+## Adapters Used
+
+None.
+
+## State And Manifest Behavior
+
+Destination state URL fields and configured link URLs share the same URL validation policy. Source manifests are not involved.
+
+## Skip And Resume Behavior
+
+None.
+
+## Failure Behavior
+
+Validation rejects parse failures, non-HTTP(S) schemes, missing hosts, query strings, and fragments.
+
+## Tests To Inspect
+
+- `internal/link/url_test.go`
+- callers in `internal/config`, `internal/state`, and `internal/publish`
+
+## Architectural Invariants
+
+- Configured and persisted link URLs use one validation policy.
+- URL construction remains outside `internal/link`.
+- Callers own field-specific error context.

docs/internal/notify.md

@@ -1,35 +1,48 @@
-# Notify
+# Notification Internals
+
+Audience: developers and LLM coding agents changing `internal/notify` or app notification wiring.
 
 ## Purpose
 
-`internal/notify` defines the internal notification interface used by the application runner.
+`internal/notify` defines the notification interface used by app orchestration after successful destination publication or replacement.
 
-## Inputs and outputs
+## Inputs And Outputs
 
-Input is a notification event containing pipeline id, destination id, bundle id, bundle path, action, and output metadata. The interface returns an error so app orchestration can treat notification failures as destination failures.
-
-## Current behavior
-
-The implemented notifier is a no-op. It is invoked only after a successful publish or replacement. Dry-run, skipped destinations, and failed destinations do not invoke it.
-
-## Failure behavior
-
-`notify.Noop` always succeeds unless the context is already canceled. If a configured notifier returns an error, `internal/app` records that destination as failed and continues with remaining destinations.
+Input is a context and notification event containing pipeline id, destination id, bundle id, bundle path, action, and output metadata. Output is an error that app orchestration can record as a destination-scoped failure.
 
 ## Boundaries
 
-External notification adapters and user-facing notification configuration are outside current behavior.
+Only the no-op notifier exists in the repository. The package does not load config, send network requests, write destination state, publish files, or own run reporting.
 
-## Tests
+## Config Fields Used
 
-Before changing notification behavior, inspect:
+None.
+
+## Adapters Used
+
+None.
+
+## State And Manifest Behavior
+
+Notification events carry output metadata projected from publish plans. The package does not inspect source manifests or destination state.
+
+## Skip And Resume Behavior
+
+Dry-run, skipped destinations, failed destinations, and planning failures do not notify. The no-op notifier has no durable state.
+
+## Failure Behavior
+
+`notify.Noop` returns the context error when the context is canceled; otherwise it succeeds. If another notifier implementation returns an error, `internal/app` records the affected destination as failed and continues with remaining destinations where applicable.
+
+## Tests To Inspect
 
 - `internal/notify`
-- `internal/app/run_test.go`
+- `internal/app/run_notify.go`
+- notification coverage in `internal/app/run_test.go`
 
-## Invariants
+## Architectural Invariants
 
-- Notifications are emitted only after successful publish or replacement execution.
+- Notifications occur only after successful publish or replacement execution.
 - Dry-run never notifies.
 - Skipped and failed destinations never notify.
 - The default app notifier is `notify.Noop`.

docs/internal/publish.md

@@ -1,43 +1,63 @@
-# Publish
+# Publish Internals
+
+Audience: developers and LLM coding agents changing `internal/publish`.
 
 ## Purpose
 
-`internal/publish` plans and executes publication for one validated source bundle and one destination.
+`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns destination comparison mapping, output selection, URL planning, managed cleanup selection, replacement safety, and destination state projection.
 
-## Inputs and outputs
+## Inputs And Outputs
 
-Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, publish policy, transform policy, transformer resolver, transfer policy, destination bundle path, and existing destination state.
+Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, transformer resolver, transfer policy, distributor version, and force flag.
 
-Output is a plan with an action, reason, and selected source or generated outputs. Execution writes selected source files, generated files, and `.distributor.json` for publish or replacement actions.
-
-## Actions
-
-Supported actions are `publish_new`, `replace_older`, `skip_same`, `skip_destination_newer`, `fail_conflict`, and `fail_unmanaged`.
-
-## Failure behavior
-
-Planning fails when request fields are incomplete, publish and transform policies are invalid, selected outputs collide, HTML output is requested without Markdown inputs, destination state is invalid, destination content is unmanaged, or transfer policy maps the comparison outcome to failure.
-
-Execution fails if a write, delete, state serialization, or context check fails. Outputs written during a failed publish attempt are cleaned up through managed deletion where possible.
+Output from planning is a `Plan` with action, reason, destination identity, selected outputs, optional existing state, optional primary URL, and force metadata. Execution writes selected source outputs, generated outputs, and `.distributor.json` for executable publish or replacement actions.
 
 ## Boundaries
 
-The current implementation publishes source files and Markdown-to-HTML sidecar outputs. Backend behavior is supplied through `internal/storage`; app runtime currently supplies local backends.
+The package does not parse CLI flags, load config files, open concrete adapters, discover source bundles, select fixed-path bundle candidates, register transforms, or render command output. The app layer supplies validated request data and concrete dependencies.
 
-The package uses `internal/state` for destination comparison, `internal/storage` for IO, and the shared `internal/config` publish/transform policy helper for request validation. It resolves transforms through a narrow resolver supplied by the caller; concrete transform registration is owned by the app layer. It does not parse CLI flags or load config files.
+External destination state semantics are documented in `docs/integrations/destination-state.md`.
 
-## Safety
+## Config Fields Used
 
-Replacement deletes only outputs recorded in existing destination state plus `.distributor.json`. Failed local writes trigger cleanup of outputs written during the failed attempt.
+The package consumes already-defaulted config values for destination `publish`, `transform`, `links`, `transfer`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
 
-## Tests
+## Adapters Used
 
-Before changing publish behavior, inspect tests under `internal/publish` and local run tests under `internal/app`.
+The package depends on `internal/storage.Backend` for source and destination IO, and on a narrow transformer resolver interface for generated outputs. It does not import concrete storage adapters or concrete transform implementations.
 
-## Invariants
+## State And Manifest Behavior
 
-- Publish planning is deterministic for the same source, destination state, policies, and transform outputs.
-- Replacement deletes only managed paths recorded in existing state plus `.distributor.json`.
-- Publish execution writes destination state after selected outputs are written.
-- Transform implementations are resolved through an interface supplied by the caller.
-- Unmanaged destination content is never overwritten.
+Planning inspects destination state through `internal/state`, compares it with the source manifest, and maps comparison outcomes plus transfer policy into actions: `publish_new`, `replace_older`, `force_replace`, `skip_same`, `skip_destination_newer`, `fail_conflict`, or `fail_unmanaged`.
+
+Execution writes destination state after selected outputs are written. Destination state includes copied source output metadata, generated output metadata, embedded source manifest, link metadata when configured, pipeline id, destination id, and publication timestamp.
+
+## Skip And Resume Behavior
+
+`skip_same` and `skip_destination_newer` execute as no-ops. Normal replacement removes only managed output paths from existing state plus `.distributor.json`; this allows retries without broad deletion. Failed writes trigger cleanup of outputs written during that failed attempt where practical.
+
+Forced replacement is explicit per request and deletes the bounded destination bundle path before writing new outputs and state.
+
+## Failure Behavior
+
+Planning fails for incomplete requests, invalid publish/transform policy, output path collisions, invalid destination state, unmanaged destination content without force, conflict outcomes not allowed by transfer policy, unresolved transforms, invalid Markdown output selection, and invalid link URL planning.
+
+Execution fails on delete, read, transform output, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable publish or replacement actions.
+
+## Tests To Inspect
+
+- `internal/publish/*_test.go`
+- `internal/app/run_test.go`
+- `internal/state/*_test.go`
+- `internal/transform/markdown/*_test.go`
+
+## Architectural Invariants
+
+- Planning is deterministic for the same request and destination state.
+- Destination bundle paths are caller-supplied and backend-root-relative.
+- URL generation uses URL path semantics and never infers public URLs from backend config.
+- Normal replacement deletes only managed paths recorded in existing state plus `.distributor.json`.
+- Forced replacement deletes only within the supplied destination bundle path.
+- Destination state is written after selected outputs are written.
+- Transform resolution stays behind a caller-supplied interface.
+- Unmanaged content is claimed only by explicit force.

docs/internal/state.md

@@ -1,52 +1,54 @@
-# Destination State
+# Destination State Internals
+
+Audience: developers and LLM coding agents changing `internal/state`.
 
 ## Purpose
 
-`internal/state` parses, validates, writes, and compares `.distributor.json` destination state.
+`internal/state` parses, validates, serializes, and compares `.distributor.json` destination state.
 
-## Inputs and outputs
+## Inputs And Outputs
 
-Input is JSON destination state plus the current source manifest, pipeline id, destination id, and whether the destination path has unmanaged content. Output is a deterministic comparison outcome and reason.
-
-## State behavior
-
-`.distributor.json` requires:
-
-- `schema_version: 1`
-- `pipeline_id`
-- `destination_id`
-- `published_at`
-- `source.manifest`
-- `outputs`
-
-`distributor_version` is optional diagnostic metadata. `published_at` parses as RFC3339 and distributor-written state serializes it as RFC3339 UTC.
-
-The embedded `source.manifest` is validated with the same source manifest rules used by `internal/bundle`.
-
-## Outputs
-
-Each output records `path`, `kind`, `source_path`, `sha256`, and `size`. Supported output kinds are `source` and `generated`. Generated outputs require `transform`.
-
-## Comparison
-
-Comparison outcomes cover absent destination state, unmanaged destination content, invalid state, pipeline or destination mismatch, same source manifest, older destination source, newer destination source, same-created digest conflict, and different source id conflict.
-
-## Failure behavior
-
-Invalid JSON, invalid state schema, invalid embedded source manifests, unsafe output paths, unsupported output kinds, missing generated-output transform names, and mismatched pipeline or destination ids produce comparison outcomes that publish planning can turn into fail actions.
+Inputs are destination state JSON, constructed state values, current source manifest, pipeline id, destination id, and whether the destination path has content without state. Outputs are validated state values, JSON bytes, comparison outcomes, and human-readable reasons.
 
 ## Boundaries
 
-This package does not publish files, delete files, inspect storage backends, or choose transfer policy actions. Publish planning consumes these comparison outcomes later.
+The package does not inspect storage backends, mutate files, choose transfer policy, build publish outputs, generate URLs, or parse config. Publish planning consumes state comparison outcomes.
 
-## Tests
+The external destination state contract is documented in `docs/integrations/destination-state.md`.
 
-Before changing destination state behavior, inspect tests under `internal/state`.
+## Config Fields Used
 
-## Invariants
+None directly. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
+
+## Adapters Used
+
+None.
+
+## State And Manifest Behavior
+
+`.distributor.json` schema version is `1`. Required fields are `pipeline_id`, `destination_id`, `published_at`, `source.manifest`, and `outputs`. `distributor_version` and `links` are optional.
+
+Embedded source manifests are parsed and validated through `internal/bundle`, which delegates source manifest semantics to `pkg/bundle`. Output records require clean paths, `source` or `generated` kind, valid source paths, lowercase SHA-256 digests, non-negative sizes, and transform ids for generated outputs. Stored URLs must pass `internal/link` validation.
+
+## Skip And Resume Behavior
+
+Comparison is pure. It returns outcomes for absent state, unmanaged content, invalid state, pipeline/destination mismatch, same source manifest, older destination, newer destination, same-created digest conflict, and different source id conflict. It does not decide whether to skip, replace, force, or fail; publish planning maps outcomes to actions.
+
+## Failure Behavior
+
+Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid embedded manifests, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transforms, invalid URLs, invalid digests, and negative sizes.
+
+## Tests To Inspect
+
+- `internal/state/distributor_test.go`
+- `internal/state/compare_test.go`
+- `internal/publish/*_test.go`
+
+## Architectural Invariants
 
 - `.distributor.json` is the destination sentinel and state record.
-- Embedded source manifests use the same validation rules as source bundles.
+- Comparison does not mutate storage.
+- Embedded source manifests use the source bundle contract.
 - Generated outputs always record a transform id.
-- Comparison returns outcomes and reasons; it does not mutate storage.
+- Stored URLs are optional and must be absolute HTTP or HTTPS URLs when present.
 - `distributor_version` is diagnostic metadata, not a comparison key.

docs/internal/storage.md

@@ -1,53 +1,56 @@
-# Storage
+# Storage Internals
+
+Audience: developers and LLM coding agents changing `internal/storage`, storage adapters, or storage-backed callers.
 
 ## Purpose
 
-`internal/storage` defines backend-rooted logical file access for core packages. Callers use slash-separated paths relative to a configured backend root.
+`internal/storage` defines backend-rooted logical file access, path validation, typed storage errors, traversal helpers, backend registration, managed deletion targets, and test fake storage behavior.
 
-## Inputs and outputs
+## Inputs And Outputs
 
-The storage interface supports byte reads, stream reads, byte writes, stream writes, exact metadata lookup, traversal, destination emptiness checks, and guarded managed deletion.
-
-Entries report a logical path, type, and size when available. Entry types are `file`, `directory`, `symlink`, and `other`.
+Inputs are contexts, logical paths or prefixes, byte slices or readers, write options, walk options, delete options, and backend open configs. Outputs are file bytes, readers, `Entry` metadata, walk callbacks, boolean content checks, registered backends, and typed errors.
 
 ## Boundaries
 
-Core packages should depend on `internal/storage`, not adapter packages. Adapter-specific path handling stays behind backend implementations.
+Core packages depend on `internal/storage`, not concrete adapters. Adapter protocol behavior belongs in `internal/adapters/local`, `internal/adapters/ssh`, and `internal/adapters/s3`; external SSH/SFTP and S3 notes live under `docs/integrations/`.
 
-The local adapter lives in `internal/adapters/local`. Runtime backend construction is wired through the app-level backend factory and storage registry. The fake backend lives in `internal/storage/fake` for tests and is not registered for runtime use.
+Runtime backend construction and registration are owned by `internal/app`. The fake backend is for tests only.
 
-## Paths
+## Config Fields Used
 
-Logical file paths must be non-empty, relative, clean, slash-separated, and must not contain `.` or `..` segments or backslashes. Prefix paths follow the same rules, except an empty prefix means the backend root.
+The storage package does not read config directly. App adapter wiring converts config fields into backend open config values.
 
-## Failure behavior
+## Adapters Used
 
-Storage errors use typed categories such as not found, already exists, invalid path, conflict, permission, temporary, unsupported, and unknown. Callers should use helper predicates rather than matching error strings.
+Local, SSH/SFTP, and S3-compatible adapters implement `storage.Backend`. `internal/storage/fake` implements the same interface for tests.
 
-Backends may wrap implementation-specific errors, but callers should receive storage errors where practical. Traversal can stop cleanly with `ErrStopWalk`.
+## State And Manifest Behavior
 
-## Deletion
+Storage owns `.distributor.json` path helpers through `StateFileName`, `StatePath`, and `ManagedBundleTargets`. It does not parse source manifests or destination state.
 
-Backends expose guarded managed deletion only. `DeleteManagedBundle` may delete listed managed outputs plus `.distributor.json`; it does not provide broad recursive deletion.
+Logical paths are slash-separated and relative to a backend root. Prefix validation allows an empty prefix to mean the backend root; file path validation requires a non-empty path.
 
-## Local and fake backends
+## Skip And Resume Behavior
 
-The local adapter maps logical paths to a configured filesystem root and keeps adapter-specific path handling behind the storage interface.
+Storage has no publication skip policy. It supplies `HasAny` for unmanaged-content checks, `DeleteManagedBundle` target construction for normal replacement cleanup, and `DeletePrefix` semantics for explicit forced replacement.
 
-The fake backend is an in-memory implementation for package tests. It is not registered for runtime use.
+## Failure Behavior
 
-## Tests
+Storage errors use typed categories: not found, already exists, not empty, invalid path, conflict, permission, temporary, unsupported, and unknown. Callers should use helper predicates instead of matching strings. Traversal can stop cleanly with `ErrStopWalk`.
 
-Before changing storage behavior, inspect tests under:
+## Tests To Inspect
 
-- `internal/storage`
-- `internal/storage/fake`
-- `internal/adapters/local`
+- `internal/storage/*_test.go`
+- `internal/storage/fake/*_test.go`
+- `internal/adapters/local/*_test.go`
+- `internal/adapters/ssh/*_test.go`
+- `internal/adapters/s3/*_test.go`
 
-## Invariants
+## Architectural Invariants
 
-- Core packages depend on `internal/storage`, not concrete adapters.
-- Logical paths are slash-separated and confined to the backend root.
-- `storage.List` uses backend traversal and returns deterministic entries.
-- Managed deletion is limited to recorded outputs plus `.distributor.json`.
-- Runtime backend registration is owned by `internal/app`.
+- Logical paths are clean relative slash-separated paths confined to the backend root.
+- Core packages never import concrete adapters.
+- `storage.List` returns deterministic sorted entries.
+- Managed deletion targets are recorded outputs plus `.distributor.json`.
+- Prefix deletion is bounded to the requested logical prefix.
+- Runtime registration remains app-owned.

docs/internal/transform.md

@@ -1,46 +1,52 @@
-# Transform
+# Transform Internals
+
+Audience: developers and LLM coding agents changing `internal/transform` or transform implementations.
 
 ## Purpose
 
-`internal/transform` defines generated publication artifacts. `internal/transform/markdown` implements Markdown-to-HTML sidecar generation.
+`internal/transform` defines generated publication artifacts, transform request/response types, transform registry behavior, and transform identifiers. `internal/transform/markdown` implements Markdown-to-HTML generation.
 
-## Inputs and outputs
+## Inputs And Outputs
 
-Inputs are a validated source bundle and source backend. Outputs include destination path, source path, transform id, generated bytes, SHA-256, and size.
-
-## Registry
-
-`internal/transform` defines the transform interface and registry. The app layer registers the Markdown implementation; publish planning receives only a resolver.
-
-## Markdown behavior
-
-Markdown files ending in `.md` generate `.html` files in the same logical directory. Non-Markdown files do not generate outputs. Raw HTML embedded in Markdown is not passed through by the renderer.
-
-Generated HTML is deterministic for the same source content and transform configuration.
-
-See `docs/integrations/markdown.md` for the Goldmark integration contract.
-
-## Failure behavior
-
-Transform resolution fails when a requested transform id is not registered. Markdown rendering fails when the source file cannot be read or rendered. Publish planning fails when HTML output is requested and the selected transform produces no outputs for a bundle.
+Inputs are a validated source bundle, source storage backend, and transform options supplied by publish planning. Outputs are generated artifact records containing destination path, source path, transform id, generated bytes, SHA-256 digest, and byte size.
 
 ## Boundaries
 
-Transforms do not publish files, mutate source bundles, or write destination state. Publish planning selects and writes transform outputs.
+Transforms do not mutate source bundles, publish files, write destination state, choose destination actions, parse config, or inspect destinations. Publish planning decides whether generated outputs are selected and writes destination state later.
 
-The app layer owns default transform registration. The transform package does not import concrete transform implementations.
+The Goldmark renderer contract is documented in `docs/integrations/markdown.md`.
 
-## Tests
+## Config Fields Used
 
-Before changing transform behavior, inspect tests under:
+Transform packages do not read config directly. Publish planning passes effective `transform.markdown_to_html.mode` and `transform.markdown_to_html.input` values.
 
-- `internal/transform`
-- `internal/transform/markdown`
+## Adapters Used
 
-## Invariants
+Transforms read source files through `internal/storage.Backend`. The Markdown implementation uses `github.com/yuin/goldmark` for rendering.
+
+## State And Manifest Behavior
+
+Transform outputs carry metadata later projected into destination state. Markdown sidecar mode renders manifest-listed `.md` files to same-directory `.html` outputs. Markdown index mode renders one selected Markdown source to `index.html`.
+
+## Skip And Resume Behavior
+
+Transforms have no skip/resume state. They are deterministic for the same source bytes and transform options.
+
+## Failure Behavior
+
+Registry registration fails for empty names, nil transformers, and duplicate names. Transform resolution fails when publish planning requests an unregistered transform. Markdown rendering fails on source read errors, renderer errors, unsafe configured input, missing manifest input, non-Markdown input, ambiguous index input, or absent Markdown inputs.
+
+## Tests To Inspect
+
+- `internal/transform/*_test.go`
+- `internal/transform/markdown/*_test.go`
+- `internal/publish/*_test.go`
+
+## Architectural Invariants
 
 - Source bundle files are never mutated by transforms.
 - Generated outputs record destination path, source path, transform id, SHA-256, and size.
-- Markdown sidecar naming changes only the `.md` extension to `.html`.
-- Non-Markdown source files do not generate Markdown outputs.
+- Markdown sidecar naming changes only the `.md` suffix to `.html`.
+- Markdown index mode always writes `index.html`.
+- Non-Markdown source files do not generate sidecar outputs.
 - Transform registration stays outside publish planning.

docs/operations.md

@@ -1,85 +1,229 @@
 # Distributor Operations
 
+Audience: administrators and operators who run `distributor`, publish bundles, operate the HTTP upload service, or recover from failed runs.
+
+This document covers operating workflows, storage layout, safety behavior, and recovery. Command syntax lives in [CLI](cli.md), configuration fields live in [Configuration](config.md), symptom-specific fixes live in [Troubleshooting](troubleshooting.md), and external contracts live under [Integrations](integrations/source-bundle.md).
+
 ## Normal Workflow
 
-Validate a source bundle:
+Validate a producer bundle before publishing:
 
 ```sh
 go run ./cmd/distributor validate examples/source-bundle
 ```
 
-Preview a local publication:
+Preview a configured run before writing destination content:
 
 ```sh
 go run ./cmd/distributor run --config examples/local-publish.yml --dry-run
 ```
 
-Run the local publication:
+Publish after reviewing the preview:
 
 ```sh
 go run ./cmd/distributor run --config examples/local-publish.yml
 ```
 
-Run the local HTML publication:
+Use JSON output for automation:
 
 ```sh
-go run ./cmd/distributor run --config examples/local-html.yml
+go run ./cmd/distributor run --config examples/fan-out.yml --dry-run --format json
 ```
 
-Preview local fan-out publication:
+Use configured source diagnostics when the source is defined in YAML and may be local, SSH/SFTP, or S3-compatible storage:
 
 ```sh
-go run ./cmd/distributor run --config examples/fan-out.yml --dry-run
+go run ./cmd/distributor validate --config examples/local-publish.yml --pipeline example-source-bundle
+go run ./cmd/distributor inspect --config examples/local-publish.yml --pipeline example-source-bundle --format json
 ```
 
-## Filesystem Layout
+Remote examples under `examples/ssh-destination.yml` and `examples/s3-destination.yml` are load-tested templates. Edit their endpoint, path, key, bucket, prefix, and credential values for storage you control before running them.
 
-Source bundles are discovered beneath the configured local source root. Each bundle is a directory containing `manifest.json`.
+## Filesystem And Storage Layout
 
-Destination bundle paths preserve the source bundle path relative to the source root. A source bundle at the source root publishes to the destination root. A source bundle under `daily/` publishes under `daily/` at each destination.
+A source bundle is a directory containing `manifest.json` and every file listed in that manifest. See [Source Bundle Contract](integrations/source-bundle.md). Source discovery walks beneath the configured source backend root and finds bundle directories.
 
-The maintained examples write under `workspace/`, which is ignored by Git.
+Each destination has its own backend root:
 
-## Destination State
+- Local destinations use the configured local `path`.
+- SSH/SFTP destinations use the configured remote `path`.
+- S3-compatible destinations use the configured `bucket` plus optional `prefix`.
 
-Each published destination bundle contains `.distributor.json`. This file is the managed sentinel and destination state record. It stores:
+Destination path mapping controls where each source bundle is published beneath the destination root:
 
-- pipeline and destination identity;
-- publication timestamp;
-- source manifest used for publication;
-- copied source output metadata;
-- generated output metadata.
+- `preserve_relative` publishes each source bundle at the same source-root-relative path.
+- `fixed` publishes one selected source bundle at the destination root.
+
+Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple bundles have the same timestamp, the source-root-relative bundle path in ascending order wins.
+
+Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination state record. It records the pipeline id, destination id, publication time, source manifest, copied outputs, generated outputs, and optional public URL metadata.
 
 `manifest.json` from the source bundle is not copied as destination state.
 
-Do not edit `.distributor.json` by hand during normal operation. If it is missing or invalid while destination files remain, `distributor` treats the destination as unmanaged or conflicted.
+## Destination State And Retry Behavior
 
-## Dry Runs
+`distributor` compares the source manifest to destination `.distributor.json` before writing:
 
-`--dry-run` loads and validates config, discovers source bundles, inspects destination state, plans outputs, and prints summary lines. It does not write output files or destination state.
+- No destination state and no destination content: publish new outputs.
+- Matching destination state: skip as already published.
+- Older destination state for the same source id: replace if transfer policy allows it.
+- Newer destination state: skip by default.
+- Invalid destination state, identity mismatch, different source id, or same-created digest mismatch: fail by default.
+- Content without `.distributor.json`: fail as unmanaged content by default.
 
-Dry-run output is useful before publishing to confirm actions such as `publish_new`, `replace_older`, `skip_same`, and `skip_destination_newer`.
+Normal replacement deletes only managed output paths recorded in `.distributor.json` plus the state file, then verifies the destination bundle path is empty before writing new outputs and state.
 
-## Retry and Replacement Behavior
+If a write fails after some outputs were written, `distributor` attempts to delete outputs from that failed attempt so a retry does not treat partial outputs as unmanaged content. Operators should still inspect the destination after a failed write before retrying.
 
-If a destination has matching `.distributor.json`, publication skips it as already published.
+Fan-out destinations are independent. If one destination fails after planning or execution begins, later destinations are still attempted. The command exits non-zero if any destination failed.
 
-If destination state is older than the source manifest and transfer policy allows replacement, publication deletes only managed outputs recorded in `.distributor.json` plus the state file, then writes the new outputs and state.
+## Dry Runs And Output Review
 
-If destination state is newer than the source manifest, the default behavior is to skip. If destination state has the same source id and created timestamp but a different digest, publication fails as a conflict.
+`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, or SSH `known_hosts` entries.
 
-If a destination path has files but no valid `.distributor.json`, publication fails as unmanaged content. There is no force overwrite option.
+Review these action labels before publishing:
 
-## Failure Handling
+- `publish_new`: destination is empty and unmanaged.
+- `replace_older`: destination state is older than the source.
+- `skip_same`: destination state already matches the source.
+- `skip_destination_newer`: destination state is newer than the source and is skipped.
+- `force_replace`: destructive replacement selected because `--force` is present and policy permits it.
+- `error`: planning or execution failed for that destination.
 
-If one destination fails in a fan-out run, independent later destinations are still planned and executed. The command exits non-zero after printing the final status if any destination failed.
+Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced.
 
-If a write fails during local publication, `distributor` attempts to remove outputs written during that failed attempt so a retry does not see those partial outputs as unmanaged destination content.
+JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document.
 
-After a successful publish or replacement, the internal notifier hook runs. The current default notifier is a no-op. Skipped destinations do not invoke it.
+## Forced Replacement Workflow
 
-## Caveats
+Use `--force` only after a dry run shows the intended bounded `force_replace` action:
 
-Only local-to-local execution is available. SSH execution, S3 execution, external notification adapters, and force overwrite behavior are unavailable.
+```sh
+go run ./cmd/distributor run --config <config-path> --dry-run --force
+go run ./cmd/distributor run --config <config-path> --force
+```
 
-For symptom-oriented fixes, see [troubleshooting](troubleshooting.md). For config details, see [configuration](config.md). For command syntax, see [CLI](cli.md).
+Forced replacement can claim unmanaged non-empty destination paths. State conflicts require both `--force` and transfer policy that permits replacement:
+
+- newer destination state requires `transfer.on_destination_newer: replace`;
+- conflict outcomes require `transfer.on_conflict: replace`.
+
+Forced replacement deletes the current destination bundle path before writing outputs and state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the backend root, so a forced replacement can clear that configured root.
+
+`--force` applies only to the current invocation. There is no config field that enables forced replacement by default.
+
+## HTTP Upload Operation
+
+The [HTTP Upload API Contract](integrations/http-upload.md) defines request and response details. `distributor serve` runs the HTTP upload API for pipelines whose source backend is `http_upload`. Each bearer token maps to exactly one configured upload pipeline. Token values come from the process environment or `secrets.directory`, not from YAML literal values.
+
+Start the maintained local example:
+
+```sh
+DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN=<token> \
+  go run ./cmd/distributor serve --config examples/http-upload-local.yml
+```
+
+Readiness:
+
+```sh
+curl http://127.0.0.1:8080/healthz
+```
+
+Upload one tar or tar.gz source bundle archive:
+
+```sh
+curl -X POST http://127.0.0.1:8080/upload \
+  -H "Authorization: Bearer $DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN" \
+  -H "Content-Type: application/gzip" \
+  --data-binary @bundle.tar.gz
+```
+
+For safe producer retries, include an idempotency key that is stable for the producer operation:
+
+```sh
+curl -X POST http://127.0.0.1:8080/upload \
+  -H "Authorization: Bearer $DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN" \
+  -H "Content-Type: application/gzip" \
+  -H "Idempotency-Key: producer.run.20260604T120000Z" \
+  --data-binary @bundle.tar.gz
+```
+
+Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. The package sends `Idempotency-Key` on every upload, derives `/upload` from the configured endpoint, and reuses the same key and replayable request body for safe retries:
+
+```go
+client, err := upload.NewClient(upload.ClientOptions{
+	Endpoint: "http://127.0.0.1:8080",
+	Token:    token,
+})
+result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
+	Root:           "examples/source-bundle",
+	IdempotencyKey: "producer.run.20260604T120000Z",
+})
+```
+
+The maintained example client uses the local upload server and reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`. It generates an idempotency key by default; set `DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY` when a retry must be stable across separate process runs.
+
+```sh
+go run ./examples/upload-client
+```
+
+Accepted uploads return after the archive is staged and validated:
+
+```json
+{"run_id":"example-http-upload.20260604T120000Z.abcdef12","status":"accepted"}
+```
+
+Poll status while the in-memory record is retained:
+
+```sh
+curl http://127.0.0.1:8080/runs/<run-id>
+```
+
+Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Completed records expire after `server.http.retention`. Expiration removes committed staged bundle directories for completed uploads. Restarting the process clears upload status, queue state, and in-memory records.
+
+Upload admission is bounded by `server.http.queue_size`. Publication concurrency is bounded by `server.http.max_concurrency`, and the coordinator does not run two uploads for the same pipeline at the same time.
+
+`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the authenticated pipeline. Reusing the same key with the same normalized source manifest returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`. If another request with the same key is still being staged before its manifest is known, the server returns a retryable `409 Conflict`. Idempotency records are memory-only and expire with completed upload status records.
+
+The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size. The implementation also caps extracted file count.
+
+The default bind address is private loopback. Put TLS, public routing, rate limiting, and external access policy in a reverse proxy or deployment layer.
+
+## Remote Backend Notes
+
+### SSH/SFTP
+
+SSH execution uses native SFTP. See [SSH/SFTP Integration](integrations/ssh-sftp.md). It does not shell out to `ssh`, `scp`, or `rsync`.
+
+Configure `ssh_key_file`, an SSH agent, or both. Agent identities are attempted first, followed by the configured key file. YAML password authentication is not supported.
+
+The default host key policy is `accept-new`. During dry runs, unknown host keys may be accepted for the current connection but are not persisted. Changed host keys are fatal for `strict` and `accept-new`. `host_key_policy: off` disables host key checking and should be limited to controlled test environments.
+
+### S3-Compatible Storage
+
+S3 execution uses the AWS SDK for Go v2. See [S3-Compatible Storage Integration](integrations/s3.md). Configure an endpoint, bucket, optional prefix, optional region, optional path-style setting, and optional explicit credential variable names.
+
+When explicit S3 credential variable names are configured, both must resolve to non-empty values through the process environment or `secrets.directory`. When omitted, the AWS SDK default credential chain is used as-is.
+
+Normal replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Forced replacement deletes objects under the bounded destination bundle prefix. Distributor does not manage bucket versioning or delete markers.
+
+## Secrets Operation
+
+`secrets.directory` is loaded during `run`, `serve`, and configured-source `validate` or `inspect` before credential-consuming work starts. If the directory is missing, unreadable, or contains an invalid secret filename, the command fails before storage work starts.
+
+Real process environment values take precedence over files with the same name. If the values differ and stdout is enabled, commands emit a warning naming the ignored secret variable without printing either value. The process environment is not modified.
+
+## Cleanup And Recovery
+
+Use these recovery boundaries:
+
+- For source validation failures, regenerate the source bundle and manifest together.
+- For an empty or missing destination, rerun after fixing config or storage access.
+- For unmanaged destination content, move unrelated files aside or use a different destination path before publishing.
+- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`.
+- For state conflicts, verify the source, pipeline, destination, and existing `.distributor.json` before considering `--force`.
+- For HTTP upload failures, inspect `/runs/<run-id>` while retained; after expiry or restart, rely on destination state and logs/output from the publishing run.
+
+Do not edit `.distributor.json` during normal recovery. Treat it as the managed state record used for comparison and safe cleanup.
+
+For symptom-specific fixes, see [Troubleshooting](troubleshooting.md).

docs/policy/architecture.md

@@ -22,19 +22,21 @@ The current core workflow is:
 2. open the source backend;
 3. discover source bundles beneath the source root;
 4. validate each source bundle and its `manifest.json`;
-5. for each configured destination, inspect destination state;
-6. compare source state to destination state;
-7. build a publish plan;
-8. optionally transform Markdown to HTML for that destination;
-9. publish selected source and generated artifacts;
-10. write `.distributor.json` as the destination sentinel/state file;
-11. run the notification hook, which is a no-op in the MVP.
+5. select the source bundle or bundles for each destination according to that destination's path mapping policy;
+6. open each destination backend independently;
+7. inspect destination state at the resolved destination bundle path;
+8. compare source state to destination state;
+9. build a publish plan that selects source files, generated files, destination paths, and optional public URLs;
+10. optionally transform Markdown to HTML for that destination;
+11. publish selected source and generated artifacts;
+12. write `.distributor.json` as the destination sentinel/state file;
+13. run the notification hook, whose default implementation is currently a no-op.
 
 ## Pipeline Model
 
 A pipeline has exactly one source and one or more destinations.
 
-The source is discovered and validated once. Each destination has independent backend configuration, publication policy, transform policy, replacement behavior, state, and notification behavior.
+The source is discovered and validated once. Each destination has independent backend configuration, path mapping, publication policy, transform policy, public link policy, replacement behavior, state, and notification behavior.
 
 The pipeline model is fan-out by design:
 
@@ -53,7 +55,7 @@ A source bundle is a directory containing `manifest.json`.
 
 `manifest.json` is the sole producer-to-`distributor` contract. `distributor` must not rely on producer-specific work directory layouts, filenames, metadata, or conventions outside the configured source root and the source manifest.
 
-The MVP source manifest schema is intentionally minimal:
+The source manifest schema is intentionally minimal:
 
 ```json
 {
@@ -73,7 +75,7 @@ The MVP source manifest schema is intentionally minimal:
 
 Required fields:
 
-- `schema_version`: source manifest schema version. MVP value: `1`.
+- `schema_version`: source manifest schema version. Current value: `1`.
 - `id`: stable bundle identifier.
 - `digest`: SHA-256 digest for the listed files.
 - `created`: RFC3339 timestamp. UTC is preferred; explicit offsets are allowed.
@@ -101,6 +103,7 @@ Each destination bundle path is managed by `.distributor.json`. This file is bot
 - the normalized source manifest used for publication;
 - metadata for copied source outputs;
 - metadata for generated outputs, such as HTML files;
+- optional URL metadata for published outputs;
 - any additional metadata required by `distributor`.
 
 A representative destination state file is:
@@ -127,14 +130,18 @@ A representative destination state file is:
       ]
     }
   },
+  "links": {
+    "primary_url": "https://reports.example.com/weather-daily/"
+  },
   "outputs": [
     {
-      "path": "report.html",
+      "path": "index.html",
       "kind": "generated",
       "source_path": "report.md",
       "transform": "markdown_to_html",
       "sha256": "sha256:...",
-      "size": 23456
+      "size": 23456,
+      "url": "https://reports.example.com/weather-daily/"
     }
   ]
 }
@@ -157,6 +164,8 @@ Transforms are configured per destination. A destination may receive source file
 
 The MVP supports only Markdown-to-HTML transformation. HTML generation must not mutate the source bundle. Generated outputs must be deterministic from the source bundle and destination transform configuration, and must be recorded in `.distributor.json`.
 
+Destination path mapping and public link generation are destination behavior. Source manifests do not declare where a bundle is published or which public URLs are recorded.
+
 The application should distinguish:
 
 - transform policy: how derived files are generated;
@@ -166,7 +175,7 @@ For example, one destination may publish source files only as a long-term archiv
 
 ## Backend Abstraction
 
-Sources and destinations use the same storage abstraction. Current runtime execution uses the local filesystem backend. Additional storage backends should be peer implementations behind the same interface, and any backend-specific execution limitation must be documented.
+Sources and destinations use the same storage abstraction. Current runtime execution uses the local filesystem, SSH/SFTP, and S3-compatible backends. Additional storage backends should be peer implementations behind the same interface, and any backend-specific execution limitation must be documented.
 
 Application logic must interact with storage through internal backend interfaces. Backend-specific behavior belongs in adapter packages. Pipeline, bundle, state, publish, and transform packages must not import service-specific or filesystem adapter implementation details.
 
@@ -187,13 +196,17 @@ Avoid dependencies for small conveniences. Do not let external dependency types
 Use this current layout unless the project has a documented reason to differ:
 
 - `cmd/distributor`: application entrypoint only.
+- `pkg/bundle`: public producer-facing source manifest model, digest logic, parsing, manifest building, complete local bundle writing, and local validation helpers.
 - `internal/app`: application orchestration and top-level use cases.
 - `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
 - `internal/config`: configuration structs, defaults, loading, precedence, and validation.
-- `internal/bundle`: source manifest parsing, source bundle discovery, source digest validation, and source bundle model.
+- `internal/bundle`: storage-backed source bundle discovery and validation over the public manifest contract.
 - `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
+- `internal/link`: shared HTTP URL validation for configured and persisted link metadata.
 - `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
 - `internal/adapters/local`: local filesystem backend.
+- `internal/adapters/ssh`: SSH/SFTP backend.
+- `internal/adapters/s3`: S3-compatible object storage backend.
 - `internal/transform`: transform interfaces, registry, planning, and shared transform models.
 - `internal/transform/markdown`: Markdown-to-HTML implementation.
 - `internal/publish`: destination planning, reconciliation, safety checks, and publish execution.
@@ -226,17 +239,20 @@ Pipeline configuration should express:
 - pipeline id;
 - one source backend;
 - one or more destinations;
+- per-destination path mapping;
 - per-destination publish policy;
 - per-destination transform policy;
+- per-destination public link policy;
 - validation behavior;
 - destination conflict/replacement behavior.
 
-## Modules, Stages, and Registries
+## Modules and Registries
 
-Each major stage should have an explicit input/output contract:
+Each major workflow step should have an explicit input/output contract:
 
 - source discovery;
 - source validation;
+- destination bundle selection;
 - destination state inspection;
 - destination comparison;
 - transform planning/execution;
@@ -272,9 +288,11 @@ If the application writes durable state, writes should be atomic where practical
 
 Code that deletes, moves, or overwrites files must use narrow, explicit paths. Avoid broad parent-directory operations. Cleanup that can cause data loss must be opt-in.
 
-`distributor` must never perform broad deletion against a configured source root or destination root. Destructive replacement may occur only inside a resolved destination bundle path when a valid `.distributor.json` confirms that the path is distributor-managed.
+`distributor` must never perform broad deletion against a configured source root. Destination deletion must be bounded to the resolved destination bundle path for the current source bundle and backend root.
 
-Replacement must be narrow, logged, test-covered, and configurable. Prefer deleting files recorded in `.distributor.json` and known generated outputs rather than blindly deleting parent directories. Backend implementations must guard against path traversal, prefix confusion, and accidental root deletion.
+Normal destructive replacement may occur only when a valid `.distributor.json` confirms that the destination bundle path is distributor-managed. Explicit forced replacement is a per-run CLI workflow for supported conflict and unmanaged-content cases; it must be dry-runnable, clearly reported, and constrained to the destination bundle path.
+
+Replacement must be narrow, reported, test-covered, and configurable. Prefer normal replacement that deletes files recorded in `.distributor.json` and known generated outputs. Forced replacement may delete a bounded destination bundle prefix only when the operator explicitly requests it. Backend implementations must guard against path traversal, prefix confusion, and accidental deletion above the configured backend root.
 
 Where practical, publish operations should use staging paths or temporary objects and promote them into place only after validation and transform steps succeed.
 
@@ -293,8 +311,10 @@ Important tests include:
 - relative path safety and path traversal rejection;
 - destination `.distributor.json` parsing and comparison;
 - same/older/newer/conflict publish decisions;
+- destination bundle path mapping;
 - destructive replacement safety checks;
 - transform output planning and metadata recording;
+- public URL planning and state metadata;
 - dry-run output;
 - local backend behavior with temporary directories;
 - fake backend behavior for storage-facing core logic.

docs/policy/development.md

@@ -6,25 +6,30 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
 ## Repository Layout
 
 - `cmd/distributor`: executable entrypoint only.
+- `pkg/bundle`: public producer-facing source manifest and local bundle writer helpers.
+- `pkg/upload`: public producer-facing HTTP upload client built on `pkg/bundle`.
 - `internal/app`: top-level use cases for `run`, `validate`, and `inspect`.
 - `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
 - `internal/config`: YAML configuration structs, loading, defaults, and validation.
-- `internal/bundle`: source bundle discovery, manifest parsing, digest calculation, and validation.
+- `internal/bundle`: storage-backed source bundle discovery and validation using the public manifest contract.
 - `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
 - `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
 - `internal/adapters/local`: local filesystem backend.
+- `internal/adapters/ssh`: SSH/SFTP backend.
+- `internal/adapters/s3`: S3-compatible object storage backend.
 - `internal/storage/fake`: in-memory backend for tests.
-- `internal/publish`: destination inspection, output planning, reconciliation, execution, and managed cleanup.
+- `internal/publish`: destination inspection, output planning, reconciliation, execution, managed cleanup, and explicit forced replacement.
 - `internal/transform`: transform interface and registry.
-- `internal/transform/markdown`: Markdown-to-HTML sidecar transform.
+- `internal/transform/markdown`: Markdown-to-HTML transform.
 - `internal/notify`: notification interface and current no-op notifier.
 - `internal/testutil`: shared test fixtures. Production code must not import this package.
 - `docs`: current user, operator, policy, internal, and roadmap documentation.
 - `examples`: copyable example configs and source bundles.
 
-Do not create new top-level package families such as `pkg`, `internal/stage`,
-`internal/modules`, or service-specific adapter directories unless the
-architecture policy or a current roadmap explicitly calls for them.
+Do not create new top-level package families such as public `pkg/...` packages
+beyond `pkg/bundle` and `pkg/upload`, generic workflow containers, or
+service-specific adapter directories unless the architecture policy or a
+current roadmap explicitly calls for them.
 
 ## Common Commands
 
@@ -41,6 +46,7 @@ go test ./internal/config
 go test ./internal/cli ./internal/app
 go test ./internal/publish ./internal/state
 go test ./internal/transform/markdown
+go test ./pkg/bundle ./pkg/upload
 ```
 
 Run the CLI against an example config:
@@ -69,9 +75,10 @@ GOCACHE=/private/tmp/distributor-gocache GOMODCACHE=/private/tmp/distributor-gom
 - Prefer package-local helpers over broad abstractions until behavior is shared by multiple packages.
 - Keep CLI parsing in `internal/cli`; business decisions belong in `internal/app`, `internal/bundle`, `internal/publish`, `internal/state`, and related core packages.
 - Keep adapter packages thin. Backend-specific filesystem or service behavior belongs in adapters; bundle, state, transform, and publish policy belongs outside adapters.
-- Preserve public CLI behavior, config semantics, manifest schema, destination state schema, and local MVP behavior unless the current task explicitly changes them.
+- Preserve public CLI behavior, config semantics, manifest schema, destination state schema, and implemented backend behavior unless the current task explicitly changes them.
 - Use `storage.DisplayPath`, `storage.StateFileName`, `storage.StatePath`, and `storage.ManagedBundleTargets` instead of duplicating those conventions.
-- Use `bundle.ValidateManifest` for normalized source manifest semantics, including embedded source manifests in destination state.
+- Use `pkg/bundle` for normalized source manifest semantics. Internal packages should reach those rules through `internal/bundle` when they also need storage-backed bundle discovery or validation.
+- Keep `pkg/upload` as a producer-facing HTTP client. It should depend on `pkg/bundle` and standard HTTP/archive primitives, not on `internal/app`, `internal/ingest`, server config, storage backends, or destination state types.
 - Use `config.ValidatePublishTransformPolicy` for publish and transform policy combinations.
 - Do not import concrete transform implementations from `internal/publish`; app-level wiring owns transform registration.
 - Do not import `internal/testutil` from production code.
@@ -82,6 +89,9 @@ The project currently depends on:
 
 - `gopkg.in/yaml.v3` for YAML configuration loading.
 - `github.com/yuin/goldmark` for Markdown rendering.
+- `golang.org/x/crypto/ssh`, `golang.org/x/crypto/ssh/agent`, and `golang.org/x/crypto/ssh/knownhosts` for native SSH support.
+- `github.com/pkg/sftp` for native SFTP support.
+- `github.com/aws/aws-sdk-go-v2/...` packages for S3-compatible storage support.
 
 Add external dependencies only when they materially improve correctness,
 security, interoperability, or implementation complexity. Avoid dependencies
@@ -100,9 +110,15 @@ When adding or changing configuration:
 5. Update `docs/config.md` in the same change if current user-visible config behavior changes.
 6. Update examples only with configs that are valid and executable for implemented behavior.
 
-Config validation may accept fields for backends that are not executable yet,
-but user-facing docs and examples must clearly state execution support. At the
-time of this policy, only the local backend is executable.
+Config validation may accept fields for roadmap backends before execution
+support exists, but user-facing docs and examples must clearly state execution
+support. Runtime executable backends are local, SSH, and S3.
+
+Credential-consuming code must use the config-owned environment resolver for
+explicit credential environment variable references. Do not call `os.Getenv`
+directly for backend credentials, because `secrets.directory` values are
+intentionally available through the resolver without mutating the process
+environment.
 
 ## CLI Changes
 
@@ -116,8 +132,9 @@ When adding or changing commands or flags:
 3. Add or update CLI tests in `internal/cli`.
 4. Update `docs/cli.md` if syntax, flags, output expectations, or workflows change.
 
-`validate` and `inspect` are local path commands. `run` loads configured
-pipelines and currently executes local backends only.
+`validate` and `inspect` support a local path shortcut and configured
+source-only diagnostics. `run` loads configured pipelines and executes local,
+SSH, and S3 backends.
 
 ## Storage Backends
 
@@ -133,8 +150,8 @@ When adding a backend:
 5. Add focused adapter tests and app-level wiring tests.
 6. Update user docs, operations docs, examples, and internal docs only for behavior that is actually implemented.
 
-Do not document SSH/SFTP or S3 execution as available until corresponding
-adapter packages and app wiring exist.
+Do not document future backend execution as available until the corresponding
+adapter package and app wiring exist.
 
 ## Transforms
 
@@ -157,17 +174,31 @@ Test close to the behavior being changed:
 - Use `internal/app` and `internal/cli` tests for user-facing workflows.
 - Use `internal/testutil` for shared valid fixtures only; keep edge cases near the package under test.
 - Run `go test ./...` after cross-package changes or documentation/example changes tied to tests.
+- Run `go test ./pkg/bundle ./pkg/upload` after changing producer-facing bundle or upload APIs.
+
+Live integration tests must be opt-in and skipped during normal `go test ./...`
+unless their required environment variables are set. Test-only environment
+variables must use this prefix shape:
+
+```text
+DISTRIBUTOR_TEST_<BACKEND>_*
+```
+
+Examples include `DISTRIBUTOR_TEST_SSH_HOST` and
+`DISTRIBUTOR_TEST_S3_ENDPOINT`. Do not use production credential variable names
+for test-only controls.
 
 ## Examples
 
 Examples under `examples/` must be valid, maintained, and free of secrets.
-They should be copyable for implemented behavior. Do not leave examples that
-look runnable but require unsupported backend execution.
+They should be copyable for implemented behavior. Remote examples must use
+placeholders or environment variables for endpoint and credential material.
 
 When changing examples:
 
 1. Keep paths relative to the repository where practical.
-2. Prefer local examples until remote backend support exists.
+2. Keep local examples runnable without external services; gate remote examples
+   behind user-provided endpoints and credentials.
 3. Run `go test ./internal/config` because config tests load examples.
 4. Update README, CLI, or config docs if links or recommended workflows change.
 

docs/policy/documentation.md

@@ -106,7 +106,7 @@ Recommended:
 - `examples/`
 - `docs/policy/development.md`
 
-### Modular, staged, service-oriented, or orchestration application
+### Modular, service-oriented, or orchestration application
 
 Required:
 - `docs/cli.md`, if CLI-based
@@ -175,7 +175,7 @@ It should include:
 - dependency policy;
 - how to add config fields;
 - how to add CLI flags;
-- how to add stages/modules/adapters, if applicable;
+- how to add modules or adapters, if applicable;
 - how to update examples;
 - documentation update expectations.
 
@@ -216,7 +216,7 @@ Explain when commands are useful, not just their syntax.
 
 **Audience:** administrators, operators
 
-Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
+Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
 
 It should cover:
 
@@ -248,7 +248,7 @@ Each entry should include:
 
 **Audience:** developers, LLM coding agents
 
-Required for modular, staged, service-oriented, or orchestration projects.
+Required for modular, service-oriented, or orchestration projects.
 
 This directory describes implemented internal components. It is not the roadmap.
 

docs/roadmap/config.md

@@ -1,580 +0,0 @@
-# Distributor Configuration Roadmap
-
-This roadmap defines the planned `config.yml` schema for the `distributor` MVP. The goal is to support one-to-many publication pipelines where each pipeline has one source and one or more destinations. Each destination independently controls backend configuration, publication outputs, transform behavior, and replacement policy.
-
-## Configuration Goals
-
-The MVP configuration should be:
-
-- explicit enough to avoid hidden publication behavior;
-- compact enough for routine self-hosted use;
-- backend-agnostic at the pipeline layer;
-- capable of local, SSH/SFTP, and S3-compatible source and destination backends;
-- ready for future notification adapters without exposing a fake notification feature in the MVP.
-
-## Top-Level Shape
-
-```yaml
-pipelines:
-  - id: weather-daily
-    source:
-      backend: local
-      path: /var/spool/distributor/weather
-
-    validation:
-      on_digest_mismatch: fail
-
-    destinations:
-      - id: markdown-archive
-        backend: s3
-        endpoint: https://s3.example.com
-        bucket: reports
-        prefix: weather/archive
-        region: us-east-1
-        force_path_style: true
-        credentials:
-          access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
-          secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
-
-        publish:
-          source: true
-          html: false
-
-        transfer:
-          on_destination_same: skip
-          on_destination_older: replace
-          on_destination_newer: skip
-          on_conflict: fail
-
-      - id: static-site
-        backend: ssh
-        uri: ssh://deploy@example.com:22
-        path: /srv/www/weather
-
-        publish:
-          source: false
-          html: true
-
-        transform:
-          markdown_to_html:
-            enabled: true
-            mode: sidecar
-
-        transfer:
-          on_destination_same: skip
-          on_destination_older: replace
-          on_destination_newer: skip
-          on_conflict: fail
-```
-
-## Pipeline Fields
-
-Each pipeline must include:
-
-- `id`: Required stable pipeline identifier.
-- `source`: Required source backend configuration.
-- `destinations`: Required non-empty list of destination configurations.
-
-Optional pipeline-level fields:
-
-- `validation`: Source validation behavior.
-- Future: `notifications` or `notify`, when notification adapters are implemented.
-
-A pipeline has exactly one source and one or more destinations.
-
-## Pipeline ID Rules
-
-`pipelines[].id` should:
-
-- be required;
-- be unique across the config file;
-- be stable over time;
-- use a simple slug-like format, such as `weather-daily` or `dnd-session-recaps`.
-
-Recommended validation:
-
-```text
-^[a-zA-Z0-9][a-zA-Z0-9._-]*$
-```
-
-## Source Configuration
-
-`source` defines the source root where bundles are discovered.
-
-The source backend may be:
-
-- `local`;
-- `ssh`;
-- `s3`.
-
-The source is scanned for `manifest.json` files beneath the configured root.
-
-### Local source
-
-```yaml
-source:
-  backend: local
-  path: /var/spool/distributor/weather
-```
-
-Required fields:
-
-- `backend: local`
-- `path`
-
-### SSH source
-
-```yaml
-source:
-  backend: ssh
-  uri: ssh://reports@example.com:22
-  path: /var/spool/distributor/weather
-```
-
-Required fields:
-
-- `backend: ssh`
-- `uri`
-- `path`
-
-Recommended authentication behavior:
-
-- use SSH agent by default;
-- use local known_hosts validation by default;
-- support optional key file configuration later if needed;
-- do not require passwords in YAML.
-
-Optional future fields:
-
-```yaml
-known_hosts: /home/user/.ssh/known_hosts
-key_file: /home/user/.ssh/id_ed25519
-```
-
-### S3 source
-
-```yaml
-source:
-  backend: s3
-  endpoint: https://s3.example.com
-  bucket: reports
-  prefix: incoming/weather
-  region: us-east-1
-  force_path_style: true
-  credentials:
-    access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
-    secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
-```
-
-Required fields:
-
-- `backend: s3`
-- `endpoint`
-- `bucket`
-
-Optional fields:
-
-- `prefix`
-- `region`
-- `force_path_style`
-- `credentials`
-
-Credential configuration should prefer environment variables or standard SDK behavior over literal secrets in config.
-
-## Destination Configuration
-
-Each destination defines one publication target for a pipeline.
-
-Required destination fields:
-
-- `id`
-- `backend`
-- backend-specific location fields;
-- `publish`
-
-Optional destination fields:
-
-- `transform`
-- `transfer`
-
-Each destination is independently planned and published. A destination may receive source files, generated HTML, or both.
-
-## Destination ID Rules
-
-`destinations[].id` should:
-
-- be required;
-- be unique within the containing pipeline;
-- be stable over time;
-- use a slug-like format.
-
-Recommended examples:
-
-- `markdown-archive`
-- `static-site`
-- `full-mirror`
-
-## Local Destination
-
-```yaml
-destinations:
-  - id: local-static
-    backend: local
-    path: /srv/www/reports
-    publish:
-      source: false
-      html: true
-```
-
-Required fields:
-
-- `backend: local`
-- `path`
-- `publish`
-
-## SSH Destination
-
-```yaml
-destinations:
-  - id: static-site
-    backend: ssh
-    uri: ssh://deploy@example.com:22
-    path: /srv/www/weather
-    publish:
-      source: false
-      html: true
-```
-
-Required fields:
-
-- `backend: ssh`
-- `uri`
-- `path`
-- `publish`
-
-The MVP should use a native SFTP implementation rather than shelling out to `ssh`, `scp`, or `rsync`.
-
-## S3 Destination
-
-```yaml
-destinations:
-  - id: markdown-archive
-    backend: s3
-    endpoint: https://s3.example.com
-    bucket: reports
-    prefix: weather/archive
-    region: us-east-1
-    force_path_style: true
-    credentials:
-      access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
-      secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
-    publish:
-      source: true
-      html: false
-```
-
-Required fields:
-
-- `backend: s3`
-- `endpoint`
-- `bucket`
-- `publish`
-
-Optional fields:
-
-- `prefix`
-- `region`
-- `force_path_style`
-- `credentials`
-
-## Backend Configuration Normalization
-
-The config loader should normalize backend configuration into internal source and destination backend specs. Pipeline logic should not branch on backend-specific fields.
-
-Validation should catch:
-
-- missing backend names;
-- unsupported backend names;
-- missing backend-specific required fields;
-- duplicate pipeline IDs;
-- duplicate destination IDs within a pipeline;
-- empty destination lists;
-- invalid policy values.
-
-## Publication Policy
-
-`publish` controls which categories of files are written to a destination.
-
-```yaml
-publish:
-  source: true
-  html: false
-```
-
-Fields:
-
-- `source`: Publish source artifacts listed in `manifest.json`.
-- `html`: Publish HTML files generated from Markdown source artifacts.
-
-At least one of `source` or `html` must be true.
-
-Recommended defaults:
-
-```yaml
-publish:
-  source: true
-  html: false
-```
-
-No implicit HTML transformation should occur. When `publish.html` is true, `transform.markdown_to_html.enabled: true` and `mode: sidecar` are required for the MVP.
-
-## Transform Configuration
-
-For MVP, the only supported transform is Markdown to HTML.
-
-```yaml
-transform:
-  markdown_to_html:
-    enabled: true
-    mode: sidecar
-```
-
-Fields:
-
-- `enabled`: Whether Markdown-to-HTML transform is enabled.
-- `mode`: Output mode. MVP value: `sidecar`.
-
-MVP `sidecar` behavior:
-
-- each listed Markdown source file generates an HTML file with the same base path and `.html` extension;
-- `report.md` generates `report.html`;
-- generated files are destination publication artifacts;
-- source bundles are not mutated.
-
-MVP defaulting:
-
-- If `publish.html` is false, transform may be omitted.
-- If `publish.html` is true and `transform.markdown_to_html` is omitted or disabled, config validation must fail.
-
-## Validation Policy
-
-Pipeline-level validation is intentionally narrow in the MVP.
-
-```yaml
-validation:
-  on_digest_mismatch: fail
-```
-
-Supported value:
-
-- `fail`
-
-Default:
-
-```yaml
-on_digest_mismatch: fail
-```
-
-Validation should happen before any destination writes. Warning-only digest mismatch handling is deferred and must be rejected if configured.
-
-## Transfer Policy
-
-Destination-level transfer policy controls behavior after inspecting `.distributor.json` at the destination bundle path.
-
-```yaml
-transfer:
-  on_destination_same: skip
-  on_destination_older: replace
-  on_destination_newer: skip
-  on_conflict: fail
-```
-
-Supported fields:
-
-- `on_destination_same`
-- `on_destination_older`
-- `on_destination_newer`
-- `on_conflict`
-
-MVP supported values are intentionally limited by field:
-
-- `on_destination_same`: `skip` or `fail`
-- `on_destination_older`: `replace` or `fail`
-- `on_destination_newer`: `skip` or `fail`
-- `on_conflict`: `fail`
-
-Recommended MVP defaults:
-
-```yaml
-transfer:
-  on_destination_same: skip
-  on_destination_older: replace
-  on_destination_newer: skip
-  on_conflict: fail
-```
-
-Safety rule:
-
-- `replace` must never perform broad deletion against a destination root.
-- `replace` may only operate within a resolved destination bundle path and should delete only files recorded in existing `.distributor.json.outputs` plus `.distributor.json` where practical.
-- Unmanaged non-empty destination paths fail in the MVP. Force or unmanaged overwrite configuration is deferred.
-- Broader replacement values, including replacing newer destinations or conflicts, are deferred to a later explicit force-overwrite stage.
-
-## Path Mapping
-
-MVP path mapping is fixed:
-
-```text
-destination bundle path = destination root + source relative bundle path
-```
-
-Example:
-
-```text
-source root: /var/spool/reports
-source bundle: /var/spool/reports/weather/daily/brentwood/2026-05-30
-relative bundle path: weather/daily/brentwood/2026-05-30
-
-destination root: /srv/www/reports
-destination bundle path: /srv/www/reports/weather/daily/brentwood/2026-05-30
-```
-
-Future config may support explicit path mapping, but MVP should not.
-
-## Dry Run Configuration and CLI Behavior
-
-Dry-run should be a CLI flag rather than a persistent config setting.
-
-```bash
-distributor run --config config.yml --dry-run
-```
-
-Dry-run should report:
-
-- pipeline ID;
-- source backend;
-- destination ID;
-- destination backend;
-- discovered bundle ID;
-- relative bundle path;
-- planned action;
-- reason;
-- transform outputs that would be generated;
-- files that would be written or deleted.
-
-## Example: Weather Pipeline
-
-```yaml
-pipelines:
-  - id: weather-daily
-    source:
-      backend: local
-      path: /var/spool/distributor/weather
-
-    validation:
-      on_digest_mismatch: fail
-
-    destinations:
-      - id: markdown-archive
-        backend: s3
-        endpoint: https://s3.example.com
-        bucket: reports
-        prefix: weather/archive
-        region: us-east-1
-        force_path_style: true
-        credentials:
-          access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
-          secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
-        publish:
-          source: true
-          html: false
-
-      - id: static-site
-        backend: ssh
-        uri: ssh://deploy@web.example.com:22
-        path: /srv/www/weather
-        publish:
-          source: false
-          html: true
-        transform:
-          markdown_to_html:
-            enabled: true
-            mode: sidecar
-```
-
-## Example: D&D Recap Pipeline
-
-```yaml
-pipelines:
-  - id: dnd-session-recaps
-    source:
-      backend: local
-      path: /var/spool/distributor/dnd/session-recaps
-
-    destinations:
-      - id: private-markdown-archive
-        backend: s3
-        endpoint: https://s3.example.com
-        bucket: reports
-        prefix: dnd/session-recaps
-        region: us-east-1
-        force_path_style: true
-        credentials:
-          access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
-          secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
-        publish:
-          source: true
-          html: false
-
-      - id: private-html-site
-        backend: local
-        path: /srv/www/private/dnd/session-recaps
-        publish:
-          source: false
-          html: true
-        transform:
-          markdown_to_html:
-            enabled: true
-            mode: sidecar
-```
-
-## Future Notification Configuration
-
-Notification should not be exposed as a functional MVP feature unless an adapter exists.
-
-The internal pipeline may include a no-op notification stage. Future config may look like:
-
-```yaml
-notifications:
-  - id: weather-email
-    backend: email
-    after_destinations:
-      - static-site
-    subject: "Weather report published"
-```
-
-Future notification policy should require:
-
-- notification after successful relevant publication;
-- idempotency by source manifest id and digest;
-- no duplicate notification unless explicitly forced.
-
-## Implementation Stages
-
-1. Define config structs for pipelines, sources, destinations, validation, publish, transform, and transfer policies.
-2. Implement config loading and strict validation.
-3. Implement backend-specific config validation for local, SSH, and S3.
-4. Implement defaulting for validation and transfer policies.
-5. Require explicit transform configuration when `publish.html` is true.
-6. Add example config fixtures for local-to-local, local-to-SSH, local-to-S3, and fan-out scenarios.
-7. Connect config to backend registry and publish planner.
-8. Add `--pipeline` filtering for targeted runs.
-9. Add `--dry-run` output that reflects the resolved config and planned actions.
-
-## Deferred Configuration
-
-The following configuration ideas are intentionally outside the MVP:
-
-- warning-only digest mismatch handling;
-- unmanaged destination overwrite flags until the explicit force-overwrite roadmap stage;
-- force replacement of destinations with different source ids until the explicit force-overwrite roadmap stage.

docs/roadmap/contracts.md

@@ -1,398 +0,0 @@
-# Distributor Contracts Roadmap
-
-This roadmap defines the contracts that `distributor` should implement before or alongside the MVP. The goal is to make bundle validation, destination state, digest verification, and safe replacement deterministic and testable before backend-specific publication behavior is layered on top.
-
-## Purpose
-
-`distributor` publishes manifested report bundles produced by other applications. Producer applications own domain-specific report generation. `distributor` owns validation, optional transformation, destination publication, and destination state.
-
-The MVP contract has two durable files:
-
-- `manifest.json`: source-owned bundle manifest produced by the upstream application.
-- `.distributor.json`: destination-owned publication state written by `distributor`.
-
-`manifest.json` is not copied to the destination as destination state. Instead, `.distributor.json` records the normalized source manifest, generated output metadata, and distributor-owned publication metadata.
-
-All manifest and state timestamps should be serialized as RFC3339. Internal comparison should use parsed timestamp values, and distributor-written timestamps should be normalized to RFC3339 UTC.
-
-## Terminology
-
-- **Source root**: Configured root path for a pipeline source.
-- **Bundle root**: Directory beneath the source root that contains `manifest.json`.
-- **Relative bundle path**: Bundle root path relative to the source root.
-- **Destination root**: Configured root path or prefix for a destination.
-- **Destination bundle path**: Destination root plus the relative bundle path, unless a future mapping option overrides that behavior.
-- **Source artifact**: File listed in the source `manifest.json`.
-- **Generated artifact**: File created by `distributor`, such as an HTML file derived from Markdown.
-- **Destination state**: `.distributor.json` at the destination bundle path.
-
-## Source Bundle Contract
-
-A source bundle is a directory containing a `manifest.json` file. For MVP, the manifest schema is intentionally minimal.
-
-### Required `manifest.json` fields
-
-```json
-{
-  "schema_version": 1,
-  "id": "weather.daily.brentwood.2026-05-30",
-  "digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
-  "created": "2026-05-30T11:10:00Z",
-  "files": [
-    {
-      "path": "report.md",
-      "sha256": "sha256:1111111111111111111111111111111111111111111111111111111111111111",
-      "size": 12345
-    }
-  ]
-}
-```
-
-Required top-level fields:
-
-- `schema_version`: Source manifest schema version. MVP value: `1`.
-- `id`: Stable bundle identifier. Required, non-empty string.
-- `digest`: Bundle digest. Required, lowercase `sha256:<64 hex>` string.
-- `created`: Bundle creation timestamp. Required RFC3339 timestamp. UTC is preferred; explicit offsets are allowed.
-- `files`: Non-empty array of file objects.
-
-Required file fields:
-
-- `path`: Relative path from bundle root to source artifact.
-- `sha256`: Per-file digest as lowercase `sha256:<64 hex>`.
-- `size`: File size in bytes.
-
-No other source manifest fields are required for the MVP. Additional fields may be ignored unless later documented. Destination state records the normalized source manifest model, not raw unknown manifest fields.
-
-## Source Path Safety Rules
-
-For every `files[].path`:
-
-- Path must be relative.
-- Path must not be empty.
-- Path must not contain `..` segments.
-- Path must not resolve outside the bundle root.
-- Path must use slash-separated logical paths in the manifest.
-- Absolute paths are invalid.
-- Symlinks should be rejected for MVP unless a later policy deliberately supports them.
-- `manifest.json` itself should not be listed as a source artifact.
-- `.distributor.json` should not be listed as a source artifact.
-- Duplicate logical file paths are invalid after path normalization.
-
-The implementation should validate paths before reading file contents.
-
-## Digest Contract
-
-The MVP validates both per-file digests and the bundle digest.
-
-### Per-file digest
-
-For each file listed in `files`, compute:
-
-```text
-sha256(file bytes)
-```
-
-The computed digest must match `files[].sha256`.
-
-The actual file size must match `files[].size`.
-
-### Bundle digest
-
-The bundle digest is computed from the listed file records in the listed order. The canonical algorithm is:
-
-1. For each file listed in `files`, in order:
-   - validate the file path;
-   - compute the file SHA256;
-   - determine the file size.
-2. Construct a canonical JSON array containing only:
-   - `path`;
-   - `sha256`;
-   - `size`.
-3. Preserve the source manifest's file order.
-4. Encode the array deterministically with:
-   - object fields in exactly this order: `path`, `sha256`, `size`;
-   - no extra spaces;
-   - no trailing newline;
-   - lowercase `sha256:<64 hex>` digest strings.
-5. Compute `sha256(canonical JSON bytes)`.
-6. Compare the result to top-level `digest`.
-
-Conceptual canonical payload:
-
-```json
-[
-  {"path":"report.md","sha256":"sha256:...","size":12345},
-  {"path":"summary.txt","sha256":"sha256:...","size":234}
-]
-```
-
-This avoids ambiguous concatenation of file bytes while keeping the manifest small.
-
-Implementation fixtures should include at least one reference manifest and canonical payload with known per-file and bundle digests.
-
-## Digest Mismatch Behavior
-
-Digest validation is always fatal in the MVP. A digest mismatch must fail the affected bundle, pipeline, and run before any destination writes occur.
-
-Warning-only digest behavior is intentionally deferred until a later roadmap accepts transitional ingestion semantics.
-
-## Bundle Discovery Contract
-
-A source path may contain one bundle or a tree of bundles. Discovery should scan beneath the configured source root for `manifest.json` files.
-
-For each discovered manifest:
-
-- Bundle root is the directory containing `manifest.json`.
-- Relative bundle path is computed relative to source root.
-- Destination bundle path is destination root plus relative bundle path, unless a future mapping option overrides it.
-
-If nested manifests are found, the MVP should fail with a clear error unless a later policy defines nested-bundle semantics.
-
-## Destination State Contract
-
-Destinations are managed by `.distributor.json`, not by copying `manifest.json`.
-
-A destination bundle path is considered distributor-managed only when it contains a valid `.distributor.json` written by `distributor`. Force or unmanaged-overwrite behavior is not part of the MVP.
-
-### Required `.distributor.json` shape
-
-```json
-{
-  "schema_version": 1,
-  "distributor_version": "0.1.0",
-  "pipeline_id": "weather-daily",
-  "destination_id": "static-site",
-  "published_at": "2026-05-30T11:12:00Z",
-  "source": {
-    "manifest": {
-      "schema_version": 1,
-      "id": "weather.daily.brentwood.2026-05-30",
-      "digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
-      "created": "2026-05-30T11:10:00Z",
-      "files": [
-        {
-          "path": "report.md",
-          "sha256": "sha256:1111111111111111111111111111111111111111111111111111111111111111",
-          "size": 12345
-        }
-      ]
-    }
-  },
-  "outputs": [
-    {
-      "path": "report.html",
-      "kind": "generated",
-      "source_path": "report.md",
-      "transform": "markdown_to_html",
-      "sha256": "sha256:3333333333333333333333333333333333333333333333333333333333333333",
-      "size": 23456
-    }
-  ]
-}
-```
-
-Required fields:
-
-- `schema_version`: Destination state schema version. MVP value: `1`.
-- `distributor_version`: Optional diagnostic distributor version. It must not affect source comparison.
-- `pipeline_id`: Pipeline that produced the destination publication.
-- `destination_id`: Destination within the pipeline.
-- `published_at`: RFC3339 timestamp.
-- `source.manifest`: Normalized source manifest model used for this publication.
-- `outputs`: Array of files written by `distributor` for this destination.
-
-Required output fields:
-
-- `path`: Destination-relative output path within the destination bundle path.
-- `kind`: `source` or `generated`.
-- `source_path`: Source artifact path that produced this output. For copied source files, this should equal `path` unless renamed by a future feature.
-- `transform`: Transform identifier for generated files. Empty or omitted may be allowed for copied source files.
-- `sha256`: Output file digest as lowercase `sha256:<64 hex>`.
-- `size`: Output file size in bytes.
-
-## Destination Comparison Rules
-
-Destination comparison uses `.distributor.json`, not destination `manifest.json`.
-
-For a source bundle and destination bundle path:
-
-### No `.distributor.json`
-
-If no `.distributor.json` exists and the destination path is empty:
-
-- Publish normally.
-
-If no `.distributor.json` exists and the destination path is non-empty:
-
-- Fail as unmanaged content.
-
-Local destination paths are empty when the destination bundle directory does not exist or exists with no entries. S3-compatible destination prefixes are empty when no objects exist below the destination bundle prefix, ignoring objects outside that exact prefix.
-
-### Pipeline or destination mismatch
-
-If `.distributor.json` exists but its `pipeline_id` or `destination_id` differs from the current pipeline or destination config:
-
-- Fail as a conflict.
-
-### Same source manifest
-
-If `.distributor.json` exists and `source.manifest` exactly matches the current normalized source manifest:
-
-- Skip as already published.
-
-### Same source id, destination older
-
-If `.distributor.json` exists, `source.manifest.id` matches the source manifest `id`, and destination `source.manifest.created` is older than the source `created`:
-
-- Replace destination contents, subject to replacement safety rules.
-
-### Same source id, destination newer
-
-If `.distributor.json` exists, `source.manifest.id` matches the source manifest `id`, and destination `source.manifest.created` is newer than the source `created`:
-
-- Skip and log that destination is newer than source.
-
-### Same source id and created, different digest
-
-If `.distributor.json` exists, `source.manifest.id` and `created` match but `digest` differs:
-
-- Fail as a conflict.
-
-### Different source id
-
-If `.distributor.json` exists and `source.manifest.id` differs from the source manifest `id`:
-
-- Fail as a conflict.
-
-## Replacement Safety Rules
-
-Replacement is destructive and must be narrow.
-
-`distributor` must never perform broad deletion against a configured destination root.
-
-Replacement may occur only at a resolved destination bundle path when:
-
-- a valid `.distributor.json` exists at that destination bundle path; and
-- the state identifies the path as distributor-managed; and
-- the replacement decision follows the destination comparison rules.
-
-For MVP, replacement should delete only known managed outputs where practical:
-
-- files listed in existing `.distributor.json.outputs`;
-- existing `.distributor.json`;
-- empty directories created by those files, where applicable for filesystem-like backends.
-
-For S3, replacement should delete only objects under the destination bundle prefix that are listed in `.distributor.json.outputs` plus `.distributor.json`, unless a later managed-prefix deletion policy is explicitly implemented.
-
-Before any write, planned destination output paths must be checked for collisions. For example, if `report.md` is copied as a source artifact and Markdown transformation would also generate `report.html`, but `report.html` is already a source artifact or another generated output, planning must fail before writing.
-
-## Transform Output Contract
-
-Source files are canonical. Generated files are derived publication artifacts.
-
-For MVP, the only supported transform is Markdown to HTML.
-
-Recommended MVP behavior:
-
-- Transform is configured per destination.
-- Markdown source files are files listed in `manifest.json` with `.md` extension.
-- Generated HTML files are sidecars by default.
-- `report.md` generates `report.html`.
-- Source bundles are never mutated.
-- Generated outputs are recorded in `.distributor.json.outputs`.
-- Raw HTML embedded in Markdown is escaped or disabled by default for the MVP to keep generated output deterministic and conservative.
-
-Future versions may add templates, `index.html`, CSS assets, email-safe HTML, and per-file transform selection.
-
-## Destination Output Contract
-
-Each destination chooses which categories of files it receives.
-
-MVP categories:
-
-- `source`: copied source artifacts listed in `manifest.json`.
-- `html`: generated HTML derived from Markdown source artifacts.
-
-Examples:
-
-- Markdown archive: `source: true`, `html: false`.
-- Static HTML site: `source: false`, `html: true`.
-- Full mirror: `source: true`, `html: true`.
-
-Every file written to the destination must be represented in `.distributor.json.outputs`.
-
-## Atomicity and Partial Failure
-
-The MVP should prefer staging and promotion where backend semantics permit it.
-
-Minimum behavior:
-
-- Validate source before writing destination files.
-- Do not write `.distributor.json` until all configured destination outputs are successfully written.
-- If publication fails before `.distributor.json` is written, the destination must not be treated as successfully published on a later run.
-- Local publication must use staging or equivalent cleanup behavior so a failed write does not leave a confusing unmanaged destination bundle path.
-- Later cleanup may remove orphaned files, but MVP correctness should rely on `.distributor.json` as the success marker.
-
-## Example Source Bundle
-
-```text
-weather/daily/brentwood/2026-05-30/
-  manifest.json
-  report.md
-  summary.txt
-```
-
-Example manifest:
-
-```json
-{
-  "schema_version": 1,
-  "id": "weather.daily.brentwood.2026-05-30",
-  "digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
-  "created": "2026-05-30T11:10:00Z",
-  "files": [
-    {
-      "path": "report.md",
-      "sha256": "sha256:1111111111111111111111111111111111111111111111111111111111111111",
-      "size": 12345
-    },
-    {
-      "path": "summary.txt",
-      "sha256": "sha256:2222222222222222222222222222222222222222222222222222222222222222",
-      "size": 234
-    }
-  ]
-}
-```
-
-## Example Destination Bundle: HTML Only
-
-```text
-weather/daily/brentwood/2026-05-30/
-  report.html
-  .distributor.json
-```
-
-## Example Destination Bundle: Source Archive
-
-```text
-weather/daily/brentwood/2026-05-30/
-  report.md
-  summary.txt
-  .distributor.json
-```
-
-## Implementation Stages
-
-1. Define Go structs for source manifest and destination state.
-2. Implement source path validation.
-3. Implement per-file SHA256 and size validation.
-4. Implement canonical bundle digest validation.
-5. Implement source bundle discovery beneath a source root.
-6. Implement `.distributor.json` parsing and validation.
-7. Implement destination comparison rules.
-8. Implement replacement safety checks.
-9. Add fixture bundles for valid, invalid, duplicate path, older, newer, same, and conflict scenarios.
-10. Add reference canonical digest fixtures.
-11. Use the contract layer from the publish pipeline and backend adapters.

docs/roadmap/documentation.md

@@ -1,468 +0,0 @@
-# Documentation Roadmap
-
-## Purpose
-
-This roadmap defines a documentation-only refresh plan for `distributor` after the local MVP and cleanup roadmap implementation.
-
-The goal is to make current-behavior documentation concise, accurate, and compliant with `docs/policy/documentation.md` while keeping planned, aspirational, or unimplemented behavior under `docs/roadmap/`.
-
-This file is written for an LLM coding agent that will implement the documentation refresh in stages. It does not itself rewrite current user, policy, internal, or example documentation.
-
-## Repository Documentation Inventory
-
-The current documentation and examples reviewed are:
-
-| Path | Current status | Notes |
-| --- | --- | --- |
-| `README.md` | Keep and lightly update | Short, accurate orientation page with a runnable local example command. |
-| `docs/cli.md` | Keep and update against CLI tests | Covers `version`, `run`, `validate`, and `inspect`; should be checked against command help and parser tests. |
-| `docs/config.md` | Keep and tighten | Current config reference documents local execution and also accepted SSH/S3 config fields. It must clearly distinguish config validation from executable backend support. |
-| `docs/operations.md` | Keep and expand slightly | Covers local workflow, destination state, retry behavior, cleanup, fan-out failure handling, and caveats. |
-| `docs/troubleshooting.md` | Added | Covers recurring local MVP failure modes. |
-| `docs/policy/architecture.md` | Keep and clarify where needed | Development policy is broad and includes future adapter direction. Wording should not imply SSH/S3 adapters currently exist. |
-| `docs/policy/development.md` | Updated | Contains contributor and agent workflow guidance. |
-| `docs/policy/documentation.md` | Keep | Canonical documentation policy. No change required unless the policy itself changes. |
-| `docs/internal/bundle.md` | Keep and verify | Describes implemented bundle parsing, discovery, validation, and digest semantics. |
-| `docs/internal/notify.md` | Keep and verify | Accurately states current no-op notification behavior. |
-| `docs/internal/publish.md` | Keep and verify | Describes planning, execution, replacement, safety, and current local scope. |
-| `docs/internal/state.md` | Keep and verify | Describes implemented `.distributor.json` state and comparison behavior. |
-| `docs/internal/storage.md` | Keep and verify | Describes storage interface, typed errors, path rules, traversal, and managed deletion. |
-| `docs/internal/transform.md` | Keep and verify | Describes Markdown-to-HTML sidecar behavior and transform boundaries. |
-| `docs/internal/app.md` | Added | Covers orchestration, backend factory, transform registry, dry-run, summaries, and notifier handoff. |
-| `docs/internal/config.md` | Added | Covers config loading, defaults, validation, accepted-but-not-executable backends, and example tests. |
-| `docs/internal/local-backend.md` | Missing, optional | Local adapter behavior may remain in `docs/internal/storage.md`; create this only if local filesystem safety detail outgrows that doc. |
-| `docs/integrations/markdown.md` | Added | Documents current Goldmark-backed Markdown rendering behavior. |
-| `docs/roadmap/audit.md` | Historical roadmap/report | Keep under roadmap unless replaced by a new audit. |
-| `docs/roadmap/cleanup.md` | Historical or completed roadmap | Keep under roadmap; optionally add completion status in the documentation refresh. |
-| `docs/roadmap/config.md` | Roadmap | Keep as planning material; avoid linking to it as current config reference. |
-| `docs/roadmap/contracts.md` | Roadmap | Keep as planning material; current implemented contracts should be summarized in `docs/internal/` and user docs as needed. |
-| `docs/roadmap/implementation.md` | Roadmap | Keep as implementation history plus future stages; status should be clear. |
-| `docs/roadmap/packages.md` | Roadmap | Keep as planning material; current package docs belong under `docs/internal/`. |
-| `docs/roadmap/storage.md` | Roadmap | Keep as planning material; current storage contract belongs in `docs/internal/storage.md`. |
-| `examples/local-to-local.yml` | Keep | Minimal local config; load-tested. |
-| `examples/local-publish.yml` | Keep | Runnable local publication example used by README and CLI docs. |
-| `examples/local-html.yml` | Keep | Runnable local HTML example. |
-| `examples/fan-out.yml` | Updated | Local-only runnable fan-out example. |
-| `examples/source-bundle/` | Keep | Copyable valid source bundle fixture for local CLI examples. |
-
-Implementation source areas inspected for documentation truth:
-
-- CLI entrypoints: `cmd/distributor`, `internal/cli`.
-- Application orchestration: `internal/app`.
-- Config loading/defaults/validation: `internal/config`.
-- Bundle manifest, discovery, and validation: `internal/bundle`.
-- Destination state: `internal/state`.
-- Storage abstraction and local/fake backends: `internal/storage`, `internal/storage/fake`, `internal/adapters/local`.
-- Publish planning and execution: `internal/publish`.
-- Transform registry and Markdown renderer: `internal/transform`, `internal/transform/markdown`.
-- Notification hook: `internal/notify`.
-- Tests and fixtures: package tests, `internal/testutil`, `examples/`, and `internal/bundle/testdata`.
-
-Absent areas from earlier planning that should not be documented as implemented:
-
-- `internal/adapters/ssh`
-- `internal/adapters/s3`
-- `internal/stage`
-- `internal/modules`
-- `internal/validators`
-- `internal/artifacts`
-- `internal/manifest`
-- `internal/schema`
-- `internal/report`
-- `pkg`
-
-## Policy Compliance Assessment
-
-Required current-behavior docs now exist for the config-driven, stateful, modular CLI.
-
-Completed fixes from this refresh:
-
-- `docs/policy/development.md` contains real workflow guidance.
-- Non-roadmap docs are scoped to implemented behavior. SSH/S3 execution, force overwrite, and external notification adapters remain described as unavailable unless the corresponding code exists.
-- `examples/fan-out.yml` is local-only and runnable.
-
-Completed recommended fixes from this refresh:
-
-- `docs/troubleshooting.md` covers recurring local MVP failure modes.
-- `docs/internal/app.md` and `docs/internal/config.md` provide current-behavior internal references.
-- `docs/integrations/markdown.md` documents current Goldmark-backed rendering behavior.
-- Historical roadmap status notes were added where useful.
-
-No broad rewrite is needed for `README.md`, `docs/cli.md`, `docs/config.md`, or `docs/operations.md`. They are close to the implemented local MVP and should be tightened against code and tests.
-
-## Target Documentation Set
-
-### `README.md`
-
-- Audience: users, administrators, operators.
-- Purpose: short project orientation and fastest useful local command.
-- Canonical scope: project purpose, elevator pitch, one local quickstart command, links to current docs.
-- Recommended outline: title, one-sentence description, local example command, links.
-- Source-of-truth repo areas to inspect: `internal/cli`, `internal/app/run.go`, `examples/local-publish.yml`, `docs/cli.md`.
-- Acceptance criteria: command is executable in the current local MVP; README does not describe SSH, S3, force overwrite, notification adapters, or future roadmap behavior as available.
-
-### `docs/cli.md`
-
-- Audience: users, administrators, operators.
-- Purpose: canonical CLI reference.
-- Canonical scope: commands, flags, useful workflows, command output expectations, local-only limits.
-- Recommended outline: shortest useful command, command overview, flag reference, common workflows, diagnostics and recovery commands.
-- Source-of-truth repo areas to inspect: `internal/cli/*.go`, `internal/cli/*_test.go`, `internal/app/validate.go`, `internal/app/inspect.go`, `internal/app/run.go`.
-- Acceptance criteria: every documented command and flag exists; `validate` and `inspect` are documented as local path commands; `run --dry-run` output is described without over-specifying every line; unsupported remote execution is stated clearly.
-
-### `docs/config.md`
-
-- Audience: administrators, operators, advanced users.
-- Purpose: canonical configuration reference.
-- Canonical scope: config file path behavior, minimal local config, production-oriented local config, full schema, defaults, validation rules, secrets handling, links to examples.
-- Recommended outline: config file location, minimal local config, production-oriented local config, reference, defaults, secrets, examples.
-- Source-of-truth repo areas to inspect: `internal/config/config.go`, `internal/config/defaults.go`, `internal/config/load.go`, `internal/config/validate.go`, `internal/config/load_test.go`, `examples/*.yml`.
-- Acceptance criteria: fields and defaults match code; `KnownFields(true)` behavior is noted where useful; SSH/S3 fields are described as accepted by config validation but not implemented for execution; `on_digest_mismatch: warn` and unmanaged overwrite are not documented as active options.
-
-### `docs/operations.md`
-
-- Audience: administrators, operators.
-- Purpose: operating and recovery notes for the implemented local MVP.
-- Canonical scope: local workflow, filesystem layout, destination state, dry-run, retry behavior, replacement safety, failed write cleanup, caveats.
-- Recommended outline: normal workflow, filesystem layout, destination state, dry-run and planning, retry and replacement behavior, failure handling, cleanup behavior, caveats.
-- Source-of-truth repo areas to inspect: `internal/app/run.go`, `internal/publish/plan.go`, `internal/publish/execute.go`, `internal/publish/reconcile.go`, `internal/publish/safety.go`, `internal/state`, `internal/adapters/local`.
-- Acceptance criteria: describes only local-to-local operation; explains `.distributor.json` as the managed sentinel; distinguishes skip, replace, conflict, and unmanaged destination behavior; does not promise resume, remote storage, force overwrite, or external notifications.
-
-### `docs/troubleshooting.md`
-
-- Audience: administrators, operators.
-- Purpose: symptom-oriented fixes for common local MVP failures.
-- Canonical scope: implemented failure modes only.
-- Recommended outline: one entry per symptom with symptom, likely cause, diagnostic step, safe fix, and relevant link.
-- Source-of-truth repo areas to inspect: `internal/config/validate.go`, `internal/bundle/validate.go`, `internal/state/compare.go`, `internal/publish/plan.go`, `internal/publish/output.go`, CLI tests.
-- Acceptance criteria: entries are actionable and do not suggest unsafe deletion; remote backend failures are described only as unsupported execution; all fixes link to `docs/cli.md`, `docs/config.md`, or `docs/operations.md` where useful.
-
-### `docs/policy/architecture.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: development principles and architectural invariants.
-- Canonical scope: project shape, package boundaries, state/persistence philosophy, external integration philosophy, errors/logging, tests, docs, non-goals.
-- Recommended outline: keep the existing outline.
-- Source-of-truth repo areas to inspect: full package tree, implemented internal docs, roadmap files for explicitly future work.
-- Acceptance criteria: still gives long-term architecture direction, but any unimplemented adapter packages or future capabilities are worded as planned/target architecture rather than implemented behavior.
-
-### `docs/policy/development.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: contributor and agent workflow.
-- Canonical scope: repo layout, build/test commands, coding conventions, dependency policy, how to add config fields, CLI flags, backends, transforms, examples, and docs.
-- Recommended outline: repository layout, common commands, coding conventions, dependency policy, adding config fields, adding CLI flags, adding storage backends, adding transforms, updating examples, documentation expectations.
-- Source-of-truth repo areas to inspect: `go.mod`, `cmd/distributor`, `internal/*`, `examples`, tests, `docs/policy/architecture.md`, `docs/policy/documentation.md`.
-- Acceptance criteria: no placeholder content remains; commands are real; workflow guidance protects current boundaries; examples and docs update rules match policy.
-
-### `docs/internal/app.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: implemented orchestration reference.
-- Canonical scope: `Run`, `Validate`, `Inspect`, backend factory, transform registry, dry-run, per-destination fan-out, failure aggregation, notifier invocation.
-- Recommended outline: purpose, inputs and outputs, run flow, backend and transform registration, dry-run behavior, failure behavior, notification behavior, tests to inspect, invariants.
-- Source-of-truth repo areas to inspect: `internal/app/*.go`, `internal/app/*_test.go`, `internal/cli/root_test.go`.
-- Acceptance criteria: documents current local-only backend execution and the no-op default notifier; does not introduce a generic stage framework that does not exist.
-
-### `docs/internal/config.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: internal config loading/default/validation reference.
-- Canonical scope: YAML decoding, known-field rejection, defaults, validation error model, backend config shape, publish/transform validation helper, example load tests.
-- Recommended outline: purpose, inputs and outputs, loading flow, defaults, validation responsibilities, executable support boundary, tests to inspect, invariants.
-- Source-of-truth repo areas to inspect: `internal/config/*.go`, `internal/config/*_test.go`, `docs/config.md`, `examples/*.yml`.
-- Acceptance criteria: documents that SSH/S3 config validation exists while execution does not; keeps user-facing config reference canonical in `docs/config.md`.
-
-### `docs/internal/bundle.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: implemented bundle contract and validation reference.
-- Canonical scope: `manifest.json`, discovery, validation, digest semantics, storage interactions, tests.
-- Recommended outline: keep current outline and verify against code.
-- Source-of-truth repo areas to inspect: `internal/bundle`, `internal/storage`, `internal/bundle/testdata`, `examples/source-bundle`.
-- Acceptance criteria: canonical digest, duplicate paths, reserved paths, symlink rejection, RFC3339 parsing, and discovery behavior match implementation.
-
-### `docs/internal/state.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: destination state and comparison reference.
-- Canonical scope: `.distributor.json` schema, validation, output metadata, comparison outcomes.
-- Recommended outline: keep current outline and verify against code.
-- Source-of-truth repo areas to inspect: `internal/state`, `internal/publish/reconcile.go`, `internal/publish/execute.go`.
-- Acceptance criteria: state schema and comparison outcomes match implemented structs and tests; `distributor_version` is described as optional diagnostic metadata.
-
-### `docs/internal/storage.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: storage interface and backend safety reference.
-- Canonical scope: logical paths, IO methods, traversal, `HasAny`, typed errors, managed deletion, local and fake backend behavior.
-- Recommended outline: keep current outline and add any missing implemented details that matter for callers.
-- Source-of-truth repo areas to inspect: `internal/storage`, `internal/storage/fake`, `internal/adapters/local`.
-- Acceptance criteria: matches actual `Backend` interface, `WriteOptions`, `DeleteOptions`, `ErrStopWalk`, and `storage.List` helper; does not describe raw recursive delete as available.
-
-### `docs/internal/publish.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: publish planning and execution reference.
-- Canonical scope: request inputs, output planning, destination inspection, transfer policy, actions, replacement safety, cleanup on failed writes.
-- Recommended outline: keep current outline and verify against code.
-- Source-of-truth repo areas to inspect: `internal/publish`, `internal/app/run.go`, `internal/config/defaults.go`.
-- Acceptance criteria: action names match constants; transfer policy values match code; collision detection and managed deletion behavior are covered.
-
-### `docs/internal/transform.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: transform registry and Markdown transform reference.
-- Canonical scope: transform interface, registry, Markdown sidecar output, deterministic output metadata, raw HTML behavior.
-- Recommended outline: keep current outline; link to integration notes if `docs/integrations/markdown.md` is created.
-- Source-of-truth repo areas to inspect: `internal/transform`, `internal/transform/markdown`, markdown tests.
-- Acceptance criteria: `.md` to `.html` sidecar naming, skipped non-Markdown files, digest metadata, and source immutability match implementation.
-
-### `docs/internal/notify.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: notification hook reference.
-- Canonical scope: interface, no-op notifier, invocation points, non-invocation points.
-- Recommended outline: keep current outline and add tests to inspect if useful.
-- Source-of-truth repo areas to inspect: `internal/notify`, `internal/app/run.go`, `internal/app/run_test.go`.
-- Acceptance criteria: says no external notification adapters or user-facing notification config exist.
-
-### `docs/integrations/markdown.md`
-
-- Audience: developers, LLM coding agents.
-- Purpose: concise external integration note for Markdown rendering.
-- Canonical scope: Goldmark dependency, renderer defaults used by `markdown.New`, raw HTML behavior as observed in tests, deterministic wrapper template, supported output mode.
-- Recommended outline: purpose, dependency, behavior used, behavior intentionally not customized, tests to inspect, update rules.
-- Source-of-truth repo areas to inspect: `go.mod`, `internal/transform/markdown`, markdown tests.
-- Acceptance criteria: documents only the Markdown renderer behavior actually used; does not claim full CommonMark compatibility beyond Goldmark defaults.
-
-### `docs/roadmap/*.md`
-
-- Audience: maintainers, developers, LLM coding agents.
-- Purpose: future work, historical plans, accepted deferred work, and implementation prompts.
-- Canonical scope: unimplemented SSH/S3 adapters, force overwrite, notification adapters, future config fields, release readiness, historical cleanup/audit plans.
-- Recommended outline: add status notes only where useful; avoid rewriting history unless it causes confusion.
-- Source-of-truth repo areas to inspect: current implementation and each roadmap file.
-- Acceptance criteria: future work remains under `docs/roadmap/`; completed historical plans are labeled clearly enough that agents do not re-run them blindly.
-
-## File-by-File Rewrite Guidance
-
-`README.md`:
-
-- Cover: purpose, one local quickstart, links.
-- Avoid: full config schema, internal package details, remote backend promises.
-- Link to: `docs/cli.md`, `docs/config.md`, `docs/operations.md`, `docs/roadmap/`.
-- Inspect: `examples/local-publish.yml`, `internal/cli`.
-- Stale claims to remove: any implication that remote publication is implemented.
-
-`docs/cli.md`:
-
-- Cover: real command syntax and current local workflows.
-- Avoid: roadmap flags such as force overwrite or remote validation.
-- Link to: `docs/config.md`, `docs/operations.md`, `docs/troubleshooting.md` if created.
-- Inspect: `internal/cli/*_test.go`.
-- Stale claims to remove: any command or flag not present in `internal/cli`.
-
-`docs/config.md`:
-
-- Cover: current config schema and defaults.
-- Avoid: presenting SSH/S3 as executable backend support.
-- Link to: examples and operations.
-- Inspect: `internal/config/defaults.go`, `internal/config/validate.go`, `internal/config/load_test.go`.
-- Stale claims to remove: `on_digest_mismatch: warn`, unmanaged overwrite, or force replacement as active options.
-
-`docs/operations.md`:
-
-- Cover: local destination state, managed cleanup, retry behavior, fan-out failure aggregation.
-- Avoid: remote storage recovery, force cleanup, external notifier delivery.
-- Link to: `docs/troubleshooting.md` for symptom-specific fixes.
-- Inspect: `internal/app/run.go`, `internal/publish/execute.go`, `internal/state`.
-- Stale claims to remove: any resume or recovery mechanism beyond re-running after safe cleanup.
-
-`docs/policy/development.md`:
-
-- Cover: concrete contributor workflow.
-- Avoid: placeholder text and invented tools.
-- Link to: architecture and documentation policies.
-- Inspect: package tree and `go.mod`.
-- Stale claims to remove: `# Not yet implemented`.
-
-`docs/internal/*.md`:
-
-- Cover: implemented component contracts, boundaries, failure behavior, tests to inspect.
-- Avoid: roadmap package names or future stages as if they exist.
-- Link to: current user docs only when relevant.
-- Inspect: package code and tests.
-- Stale claims to remove: broad future backend behavior outside local/fake abstractions.
-
-`examples/`:
-
-- Cover: valid, maintained, copyable examples.
-- Avoid: examples that look runnable but fail because the backend execution is unsupported.
-- Link from: README, CLI docs, config docs.
-- Inspect: `internal/config/load_test.go` and optional CLI smoke commands.
-- Stale claims to remove: executable remote fan-out examples until remote backends exist.
-
-## Examples Plan
-
-Keep these examples:
-
-- `examples/source-bundle/`: valid source bundle used by local CLI examples.
-- `examples/local-to-local.yml`: minimal local config. Use in config docs as the minimal schema example.
-- `examples/local-publish.yml`: primary runnable quickstart config.
-- `examples/local-html.yml`: runnable Markdown-to-HTML example.
-
-Revise `examples/fan-out.yml` in the documentation refresh:
-
-- Preferred option: replace it with a local-only fan-out example using two local destinations, such as one source archive destination and one HTML destination under `workspace/`.
-- Alternative option: move the SSH/S3 fan-out material under a roadmap file and remove it from `examples/`.
-- Do not keep a non-roadmap example that appears copyable for execution but uses unsupported SSH/S3 execution.
-
-Future example tests should continue loading every YAML file under `examples/`. If `examples/fan-out.yml` becomes local-only, add or update a CLI/app test that exercises local fan-out behavior or rely on existing fan-out tests if they cover equivalent behavior.
-
-## Internal Documentation Plan
-
-Update existing internal docs only after checking package code and tests.
-
-Create these internal docs:
-
-- `docs/internal/app.md`: orchestration, backend factory, transform registry, dry-run behavior, fan-out, failure aggregation, notifier invocation.
-- `docs/internal/config.md`: YAML loading, defaults, validation, accepted config fields, unsupported execution boundary, example load tests.
-
-Keep and verify these internal docs:
-
-- `docs/internal/bundle.md`
-- `docs/internal/state.md`
-- `docs/internal/storage.md`
-- `docs/internal/publish.md`
-- `docs/internal/transform.md`
-- `docs/internal/notify.md`
-
-Defer these internal docs unless the corresponding implementation grows:
-
-- `docs/internal/local-backend.md`: create only if local adapter details become too long for `docs/internal/storage.md`.
-- `docs/internal/logging.md`: defer until logging has behavior beyond placeholders.
-- `docs/internal/testutil.md`: defer unless test fixture helpers become a stable contributor-facing contract.
-
-Do not create docs for packages or directories that do not exist.
-
-## Integration Documentation Plan
-
-No external service integration docs should be created for SSH, S3, or notification services until those integrations are implemented.
-
-Recommended current integration doc:
-
-- `docs/integrations/markdown.md`
-
-This should document Goldmark usage because Markdown rendering is an implemented external file-format integration with externally visible output. Keep it concise and limited to:
-
-- dependency and version source: `go.mod`;
-- renderer construction: `goldmark.New()`;
-- sidecar output path behavior owned by `internal/transform/markdown`;
-- raw HTML behavior covered by tests;
-- wrapper template behavior;
-- tests to inspect before changing renderer behavior.
-
-Do not create separate YAML integration docs unless configuration parsing behavior outgrows `docs/config.md` and `docs/internal/config.md`.
-
-## Recommended Implementation Sequence
-
-### Stage 1: Establish Documentation Status and Contributor Policy
-
-- Goal: remove the required policy placeholder and classify active versus historical documentation.
-- Files to create/update/delete/move: update `docs/policy/development.md`; optionally add short status notes to `docs/roadmap/audit.md` and `docs/roadmap/cleanup.md`.
-- Repo areas to inspect: `go.mod`, `cmd/distributor`, `internal/*`, `examples`, `docs/policy/architecture.md`, `docs/policy/documentation.md`.
-- Acceptance criteria: `docs/policy/development.md` contains real workflow guidance; no active policy doc says "Not yet implemented"; roadmap status notes do not change current behavior docs.
-- Suggested validation commands: `rg -n "Not yet implemented" docs/policy README.md docs/*.md docs/internal`; `git diff -- docs/policy/development.md docs/roadmap/audit.md docs/roadmap/cleanup.md`.
-- One prompt: yes.
-
-### Stage 2: Tighten Current User and Operator Docs
-
-- Goal: make README, CLI, config, and operations docs exactly match the local MVP.
-- Files to create/update/delete/move: update `README.md`, `docs/cli.md`, `docs/config.md`, `docs/operations.md`; create `docs/troubleshooting.md`.
-- Repo areas to inspect: `internal/cli`, `internal/app`, `internal/config`, `internal/publish`, `internal/state`, `examples`.
-- Acceptance criteria: user docs describe local execution, local validation/inspection, current config schema, current defaults, current state/retry behavior, and clear unsupported remote execution boundaries.
-- Suggested validation commands: `rg -n "force|allow_unmanaged|on_digest_mismatch: warn|warn" README.md docs/cli.md docs/config.md docs/operations.md docs/troubleshooting.md`; `rg -n "ssh|s3|remote|notification" README.md docs/cli.md docs/config.md docs/operations.md docs/troubleshooting.md`.
-- One prompt: yes.
-
-### Stage 3: Make Examples Fully Runnable or Clearly Roadmap-Only
-
-- Goal: ensure `examples/` contains implemented, copyable examples only.
-- Files to create/update/delete/move: update or replace `examples/fan-out.yml`; update links in `README.md`, `docs/cli.md`, and `docs/config.md` if needed; move remote fan-out material to a roadmap section if preserving it is useful.
-- Repo areas to inspect: `internal/config/load_test.go`, `internal/app/run_test.go`, `examples`.
-- Acceptance criteria: every example under `examples/` is valid current config and does not rely on unsupported remote execution; primary examples remain load-tested.
-- Suggested validation commands: `go test ./internal/config`; optional `go run ./cmd/distributor run --config examples/local-publish.yml --dry-run`; optional `go run ./cmd/distributor run --config examples/local-html.yml --dry-run`.
-- One prompt: yes.
-
-### Stage 4: Complete Internal Component Docs
-
-- Goal: give future agents a current-behavior internal map before remote backends are added.
-- Files to create/update/delete/move: create `docs/internal/app.md` and `docs/internal/config.md`; update existing `docs/internal/*.md` as needed.
-- Repo areas to inspect: `internal/app`, `internal/config`, `internal/bundle`, `internal/state`, `internal/storage`, `internal/storage/fake`, `internal/adapters/local`, `internal/publish`, `internal/transform`, `internal/notify`, package tests.
-- Acceptance criteria: every major implemented component has a concise doc with purpose, inputs/outputs, boundaries, failure behavior, tests to inspect, and invariants; no internal doc describes absent SSH/S3 adapters as implemented.
-- Suggested validation commands: `rg -n "internal/adapters/ssh|internal/adapters/s3|not implemented|future" docs/internal`; `git diff -- docs/internal`.
-- One prompt: yes.
-
-### Stage 5: Add Markdown Integration Notes
-
-- Goal: document the one implemented external file-format integration where behavior matters.
-- Files to create/update/delete/move: create `docs/integrations/markdown.md`; optionally link from `docs/internal/transform.md`.
-- Repo areas to inspect: `go.mod`, `internal/transform/markdown`, markdown tests.
-- Acceptance criteria: the doc is concise, version-aware through `go.mod`, and limited to current Goldmark usage and observed renderer behavior.
-- Suggested validation commands: `go test ./internal/transform/markdown`; `rg -n "Goldmark|markdown" docs/integrations docs/internal/transform.md`.
-- One prompt: yes.
-
-### Stage 6: Final Documentation Consistency Sweep
-
-- Goal: catch stale links, stale roadmap references, and unimplemented claims outside roadmap.
-- Files to create/update/delete/move: any docs touched in earlier stages.
-- Repo areas to inspect: all docs and examples.
-- Acceptance criteria: documentation is current, concise, link-consistent, and policy-compliant.
-- Suggested validation commands: `go test ./...`; `rg -n "go-application-template|maximumdirect.net|docs/architecture.md|docs/documentation.md" README.md docs examples`; `rg -n "allow_unmanaged_overwrite|on_digest_mismatch: warn" README.md docs examples`; `rg -n "not implemented|future|planned|roadmap|ssh|s3|remote" README.md docs/*.md docs/internal docs/policy examples`.
-- One prompt: yes, after the prior stages are complete.
-
-## Validation Plan
-
-Run Go tests when examples, CLI docs, config docs, or behavior-linked docs change:
-
-- `go test ./internal/config`: validates example config loading and config semantics.
-- `go test ./internal/cli ./internal/app`: validates documented command behavior and local run workflows.
-- `go test ./internal/transform/markdown`: validates documented Markdown rendering behavior.
-- `go test ./...`: final broad verification after documentation and example changes.
-
-Run grep checks:
-
-- `rg -n "Not yet implemented" docs/policy README.md docs/*.md docs/internal`
-- `rg -n "go-application-template|docs/architecture.md|docs/documentation.md" README.md docs examples`
-- `rg -n "allow_unmanaged_overwrite|on_digest_mismatch: warn" README.md docs examples`
-- `rg -n "force overwrite|--force|remote backends are implemented|notification adapters" README.md docs/*.md docs/internal docs/policy examples`
-- `rg -n "ssh|s3|remote" README.md docs/*.md docs/internal docs/policy examples`
-
-The final SSH/S3/remote grep is not expected to return zero results. Manually review every result and confirm it is either:
-
-- under `docs/roadmap/`;
-- a clearly stated unsupported-execution boundary;
-- a config-validation reference that does not imply executable support; or
-- an architecture policy statement phrased as future/target direction rather than implemented behavior.
-
-Manual review checklist:
-
-- README remains short.
-- `docs/config.md` is the only current-behavior config reference.
-- `docs/cli.md` is the only current-behavior CLI reference.
-- `docs/operations.md` covers state and recovery without unsafe deletion advice.
-- `docs/troubleshooting.md` is symptom-oriented and links to canonical docs.
-- Internal docs point to tests before changing behavior.
-- Examples are copyable and free of secrets.
-- Future work remains under `docs/roadmap/`.
-
-## Open Questions
-
-No blocking questions remain before implementing this documentation refresh.
-
-The only non-blocking choice is how to handle `examples/fan-out.yml`:
-
-- Preferred: convert it to a local-only fan-out example so `examples/` remains fully runnable.
-- Acceptable: move the current SSH/S3 fan-out example into roadmap material until remote execution exists.
-
-Use the preferred option unless a maintainer explicitly wants `examples/` to include config-validated but non-executable examples.

docs/roadmap/future.md

@@ -0,0 +1,80 @@
+# Future Roadmap
+
+This document records planned or deferred work that is not part of the current
+implementation. Current behavior is documented outside roadmap files in the
+README, integration contracts, operations guide, troubleshooting guide, and
+internal docs.
+
+## Durability And Recovery
+
+- Durable upload status persistence across process restarts.
+- Durable idempotency records across server restarts.
+- Database-backed upload queueing.
+- Recovery semantics for queued or running uploads after a restart.
+- Durable producer retry processing.
+
+## Producer Client Workflows
+
+- Durable client queues or background producer workers.
+- `UploadAndWait` helper.
+- Long-polling helper or equivalent wait workflow.
+
+## Run Control APIs
+
+- Run retry endpoints.
+- Run cancellation endpoints.
+- Run listing endpoints.
+
+## Archive And Transport Protocols
+
+- Zstandard-compressed tar archives.
+- Additional content negotiation rules for future archive formats.
+- Multipart upload support.
+- Resumable upload support.
+- Streaming upload protocols.
+
+## Destination Backends
+
+- GitHub Gist destination backend support.
+- Authentication and secret handling for GitHub API tokens.
+- Gist file mapping, update, replacement, and conflict semantics.
+- Rate-limit handling and retry behavior for GitHub API responses.
+
+## Docker Image Support
+
+- Official container image build and release workflow.
+- Runtime filesystem layout for config, secrets, staging, and local outputs.
+- Container-oriented examples for `run` and `serve`.
+- Image tagging, versioning, and upgrade guidance.
+
+## Notifications And Hooks
+
+- Email notification support for completed, failed, or partially failed
+  distribution runs.
+- SMTP configuration, authentication, secret handling, and recipient policy.
+- ntfy notification support for completed, failed, or partially failed
+  distribution runs.
+- ntfy topic, server, token, priority, and action configuration.
+- General post-distribution hook support.
+- Hook payload contract that can pass run status, summaries, destination
+  outcomes, output metadata, and public links to external tools.
+- Local executable hook adapter with bounded arguments, environment, stdin,
+  timeout, exit-code handling, and secret-redaction behavior.
+
+## Authentication And Deployment Surface
+
+- URL-token authentication for constrained clients.
+- Additional token lifecycle tooling.
+- Mutual TLS or other in-app identity mechanisms.
+- In-app TLS.
+- Public exposure defaults.
+- Browser UI.
+
+## Boundaries
+
+- `http_upload` remains source-only unless a future implementation changes that
+  contract.
+- Current upload status, queue, and idempotency state are memory-only.
+- Producers submit complete tar or gzip-compressed tar source bundles today.
+- Public access policy, TLS termination, and rate limiting belong outside
+  `distributor` unless a future implementation changes that boundary.

docs/roadmap/implementation.md

@@ -1,863 +0,0 @@
-# Distributor Implementation Roadmap
-
-This roadmap defines a staged implementation plan for the `distributor` MVP. Each stage is intended to map cleanly to one Codex implementation prompt.
-
-The roadmap assumes the project includes these planning documents before implementation begins:
-
-- `docs/policy/architecture.md`
-- `docs/policy/documentation.md`
-- `docs/roadmap/packages.md`
-- `docs/roadmap/contracts.md`
-- `docs/roadmap/config.md`
-- `docs/roadmap/storage.md`
-
-The MVP goal is a domain-agnostic bundle distributor that discovers source bundles, validates `manifest.json`, optionally transforms Markdown to HTML, publishes selected outputs to one or more destinations, and records destination state in `.distributor.json`.
-
-## Global Implementation Rules
-
-All stages should preserve these invariants:
-
-- Producer applications own source bundle creation.
-- `distributor` owns validation, transformation, publication, destination state, and future notification hooks.
-- Source bundle state is defined by `manifest.json`.
-- Destination publication state is defined by `.distributor.json`.
-- `manifest.json` is not copied to the destination as destination state.
-- Pipelines have exactly one source and one or more destinations.
-- Transform and publish policy are destination-specific.
-- Destructive replacement is allowed only inside a managed destination bundle path. Unsafe force or unmanaged overwrite behavior is deferred.
-- Dry-run behavior should be implemented before broad remote write behavior.
-- Config, bundle, state, publish planning, storage adapters, transforms, and CLI wiring should remain separate packages.
-
-Unless a stage explicitly says otherwise, each implementation prompt should:
-
-1. read the project policy and roadmap documents;
-2. implement only the current stage;
-3. add or update tests for the current stage;
-4. run the relevant test suite;
-5. update documentation only when the implemented behavior now exists;
-6. avoid implementing future roadmap stages early.
-
-## Stage 1: Project Skeleton, CLI Shell, and Baseline Tooling
-
-### Goal
-
-Create the initial Go application structure and a minimal executable `distributor` command with no business behavior beyond version/help output and placeholder commands.
-
-### Scope
-
-Implement the accepted package skeleton from `docs/roadmap/packages.md` at the level needed for compilation.
-
-Create:
-
-```text
-cmd/distributor/main.go
-internal/cli/
-internal/app/
-internal/config/
-internal/logging/
-```
-
-Initial CLI commands:
-
-- `distributor --help`
-- `distributor version`
-- `distributor run`
-- `distributor validate`
-- `distributor inspect`
-
-At this stage, `run`, `validate`, and `inspect` may return clear “not implemented” errors, but the command structure should be present.
-
-### Notes
-
-Prefer a small CLI dependency only if the project already standardizes on one. Otherwise, the standard library is acceptable for the first pass.
-
-Add a version variable that can later be set at build time.
-
-### Tests
-
-Add tests for:
-
-- command construction if testable;
-- version string behavior if exposed through a package;
-- basic package compilation.
-
-### Completion Criteria
-
-- `go test ./...` passes.
-- `go run ./cmd/distributor --help` works.
-- `go run ./cmd/distributor version` works.
-- Placeholder operational commands fail clearly and intentionally.
-
-## Stage 2: Config Schema, Loading, Defaults, and Validation
-
-### Goal
-
-Implement the MVP `config.yml` schema described in `docs/roadmap/config.md`.
-
-### Scope
-
-Create config structs for:
-
-- top-level config;
-- pipelines;
-- source backend config;
-- destination backend config;
-- validation policy;
-- publish policy;
-- transform policy;
-- transfer/replacement policy;
-- backend-specific local, SSH, and S3 fields.
-
-Support loading YAML from a file path.
-
-Implement validation for:
-
-- required top-level `pipelines`;
-- unique pipeline ids;
-- required pipeline `id`, `source`, and non-empty `destinations`;
-- unique destination ids within a pipeline;
-- supported backend names: `local`, `ssh`, `s3`;
-- required backend fields;
-- supported validation action: `fail`;
-- supported transfer actions;
-- valid `publish` policy;
-- valid Markdown-to-HTML transform config.
-
-Default behavior should match `docs/roadmap/config.md`.
-
-### CLI Integration
-
-Add `--config` to `run`.
-
-For this stage, `distributor run --config config.yml --dry-run` may only load and validate config, then print a concise summary of configured pipelines and destinations.
-
-### Tests
-
-Add unit tests for:
-
-- valid minimal local-to-local config;
-- valid fan-out config;
-- valid local, SSH, and S3 backend configs;
-- duplicate pipeline ids;
-- duplicate destination ids;
-- missing required fields;
-- unsupported backend;
-- invalid transfer action;
-- invalid validation action, including `warn`.
-
-### Completion Criteria
-
-- Config load/default/validate behavior is implemented and tested.
-- `distributor run --config <file> --dry-run` validates config and prints a summary.
-- No bundle discovery or publication occurs yet.
-
-## Stage 3: Storage Abstraction, Local Backend, and Fake Backend
-
-### Goal
-
-Introduce the storage backend abstraction before bundle validation so source discovery, validation, and publication are backend-agnostic from the start.
-
-### Scope
-
-Create:
-
-```text
-internal/storage/backend.go
-internal/storage/registry.go
-internal/storage/path.go
-internal/storage/errors.go
-internal/adapters/local/backend.go
-internal/storage/fake/
-```
-
-Implement the storage contract in `docs/roadmap/storage.md`, including backend-rooted logical paths, hybrid byte/stream IO, metadata, traversal, typed errors, managed deletion, and efficient destination emptiness helper behavior.
-
-The fake backend should exist for unit tests of config, bundle, state, and publish logic without real local, SSH, or S3 IO.
-
-### Safety Requirements
-
-The local backend must:
-
-- clean and join paths safely;
-- reject path traversal;
-- reject unsafe destructive deletion requests;
-- avoid following symlinks for source bundle files unless explicitly supported;
-- avoid deleting configured roots;
-- classify destination bundle emptiness deterministically.
-
-### Tests
-
-Add tests for:
-
-- path validation;
-- backend byte and stream read/write behavior;
-- `Stat`, `Walk`, and materialized deterministic traversal helper behavior;
-- traversal rejection;
-- symlink entry reporting and source-read rejection;
-- staged write behavior where testable;
-- typed storage errors and helper predicates;
-- managed deletion guard behavior;
-- early-stop destination emptiness helper behavior;
-- fake backend parity for core package tests.
-
-### Completion Criteria
-
-- Local backend implements the storage interface.
-- Fake backend can support bundle and publish tests without external services.
-- `go test ./...` passes.
-- No SSH or S3 implementation exists yet.
-
-## Stage 4: Source Bundle Manifest, Digest, Validation, and Discovery
-
-### Goal
-
-Implement the source bundle contract from `docs/roadmap/contracts.md` through the storage abstraction.
-
-### Scope
-
-Create:
-
-```text
-internal/bundle/manifest.go
-internal/bundle/digest.go
-internal/bundle/validate.go
-internal/bundle/discover.go
-```
-
-Implement:
-
-- parsing `manifest.json`;
-- strict required field validation, including `schema_version: 1`;
-- RFC3339 `created` parsing;
-- lowercase `sha256:<64 hex>` digest validation;
-- source file path safety checks;
-- duplicate logical file path rejection;
-- per-file SHA256 validation;
-- per-file size validation;
-- bundle digest validation using the canonical ordered file-record algorithm;
-- deterministic storage-backed bundle discovery under a source root;
-- nested manifest detection and failure.
-
-Discovery and validation should use `internal/storage` rather than direct `os` APIs. The local CLI path should be adapted to the local backend.
-
-### CLI Integration
-
-Implement:
-
-```text
-distributor validate <path>
-distributor inspect <path>
-```
-
-For local paths:
-
-- `validate` should validate either a single bundle directory or a tree containing bundles.
-- `inspect` should print a concise normalized summary of discovered bundle ids, relative paths, created timestamps, digest values, and files.
-
-### Tests
-
-Add fixture bundles under a testdata directory.
-
-Test:
-
-- valid bundle;
-- invalid JSON;
-- missing required fields;
-- invalid schema version;
-- invalid timestamp;
-- invalid digest format;
-- unsafe file paths;
-- duplicate normalized file paths;
-- missing files;
-- size mismatch;
-- per-file digest mismatch;
-- bundle digest mismatch;
-- canonical bundle digest reference fixture;
-- multiple discovered bundles in deterministic order;
-- nested manifests fail.
-
-### Completion Criteria
-
-- Storage-backed bundle validation is deterministic and well-tested.
-- `distributor validate <path>` works for local bundle fixtures.
-- `distributor inspect <path>` works for local bundle fixtures.
-- No destination publication occurs yet.
-
-## Stage 5: Destination State Contract and Comparison Logic
-
-### Goal
-
-Implement `.distributor.json` parsing, validation, and source-to-destination comparison.
-
-### Scope
-
-Create:
-
-```text
-internal/state/distributor.go
-internal/state/compare.go
-internal/state/validate.go
-```
-
-Implement the destination state schema from `docs/roadmap/contracts.md`, including:
-
-- `schema_version`;
-- optional `distributor_version`;
-- `pipeline_id`;
-- `destination_id`;
-- `published_at`;
-- embedded normalized source manifest;
-- outputs array;
-- output file metadata.
-
-Implement comparison outcomes:
-
-- destination absent;
-- destination unmanaged/non-empty;
-- destination state pipeline or destination id mismatch;
-- same source manifest;
-- same source id, destination older;
-- same source id, destination newer;
-- same source id and same created but different digest;
-- different source id;
-- invalid destination state.
-
-### Tests
-
-Add unit tests for every comparison outcome.
-
-Test validation for:
-
-- valid state;
-- missing fields;
-- invalid schema version;
-- invalid embedded source manifest;
-- invalid output metadata;
-- malformed published timestamp.
-
-Timestamps should parse RFC3339 input and distributor-written timestamps should normalize to RFC3339 UTC.
-
-### Completion Criteria
-
-- Destination state can be parsed and validated independently.
-- Source manifest to destination state comparison is deterministic and fully tested.
-- No publication execution occurs yet.
-
-## Stage 6: Publish Planning, Dry-Run, and Local-to-Local Publication Without Transform
-
-### Goal
-
-Implement the core publish planner and execute local-to-local publication for source files only.
-
-### Scope
-
-Create:
-
-```text
-internal/publish/plan.go
-internal/publish/reconcile.go
-internal/publish/safety.go
-internal/publish/output.go
-internal/publish/execute.go
-```
-
-Implement planning for one source bundle to one destination based on:
-
-- source manifest;
-- destination config;
-- publish policy;
-- transfer policy;
-- existing `.distributor.json`;
-- destination path state.
-
-Actions should include:
-
-- publish new;
-- replace older destination;
-- skip same;
-- skip destination newer;
-- fail conflict;
-- fail unmanaged destination.
-
-Implement local-to-local execution for `publish.source: true` and `publish.html: false`.
-
-Execution should:
-
-- copy listed source files selected by publish policy;
-- write `.distributor.json` with copied source output metadata;
-- avoid copying source `manifest.json` as destination state;
-- preserve relative bundle paths from source root beneath destination root;
-- detect destination output collisions before writing;
-- use staging or equivalent cleanup behavior for local writes;
-- support fan-out to multiple local destinations;
-- support dry-run without writes.
-
-### CLI Integration
-
-`distributor run --config <file>` should now execute local-to-local pipelines when configured.
-
-`--dry-run` should print the planned action for each discovered bundle and destination.
-
-### Tests
-
-Add integration-style tests using temp directories for:
-
-- new local publication;
-- no-op when destination state matches;
-- replacement when destination state is older;
-- skip when destination state is newer;
-- fail on conflict;
-- fail on unmanaged non-empty destination;
-- fail on output path collision;
-- fan-out from one source to two local destinations;
-- failed local write does not leave a destination that appears unmanaged on retry;
-- dry-run performs no writes;
-- `.distributor.json` is written correctly.
-
-### Completion Criteria
-
-- Local-to-local source-file publication works end to end.
-- Dry-run produces meaningful planned actions.
-- Destination state is authoritative.
-- No Markdown-to-HTML transform exists yet.
-
-## Stage 7: Markdown-to-HTML Transform and Destination-Specific Publish Policy
-
-### Goal
-
-Add MVP Markdown-to-HTML transformation and destination-specific source/html output selection.
-
-### Scope
-
-Create:
-
-```text
-internal/transform/transform.go
-internal/transform/registry.go
-internal/transform/plan.go
-internal/transform/markdown/markdown.go
-internal/transform/markdown/template.go
-```
-
-Implement only:
-
-```yaml
-transform:
-  markdown_to_html:
-    enabled: true
-    mode: sidecar
-```
-
-MVP sidecar behavior:
-
-- for each listed source artifact ending in `.md`, generate a same-directory `.html` sidecar;
-- preserve the original Markdown file unchanged;
-- do not generate HTML for non-Markdown files;
-- escape or disable raw HTML embedded in Markdown;
-- fail before writing when generated output paths collide with copied source outputs or other generated outputs;
-- record generated output metadata in `.distributor.json`;
-- if `publish.source: false`, do not publish source files;
-- if `publish.html: true`, publish generated HTML files;
-- if `publish.html: true` but transform is disabled or no Markdown files exist, fail with a clear error unless config later defines another behavior.
-
-Use a well-maintained Markdown renderer. Keep HTML templating minimal and deterministic.
-
-### Tests
-
-Add tests for:
-
-- Markdown sidecar generation;
-- source-only destination;
-- HTML-only destination;
-- source-plus-HTML destination;
-- no mutation of source bundle;
-- generated output metadata in `.distributor.json`;
-- failure when HTML publication is requested without transform support;
-- failure when generated HTML collides with a source artifact path;
-- raw HTML in Markdown is escaped or disabled consistently;
-- deterministic output for a fixture Markdown file.
-
-### Completion Criteria
-
-- Local-to-local publication supports source-only, HTML-only, and source-plus-HTML destinations.
-- Generated outputs are recorded in destination state.
-- Dry-run reports transform outputs that would be generated.
-
-## Stage 8: No-Op Notification Stage, Pipeline Polish, and Local MVP Checkpoint
-
-### Goal
-
-Add the internal no-op notification stage and polish orchestration around per-destination outcomes.
-
-### Scope
-
-Create:
-
-```text
-internal/notify/notify.go
-internal/notify/noop.go
-```
-
-Integrate a no-op notifier after actual successful publication or replacement. Skipped destinations should not invoke the notifier in the MVP.
-
-Clarify orchestration behavior when one destination fails. For MVP, fan-out should be deterministic and sequential. Continue planning and reporting later destinations where safe, but return non-zero if any destination fails.
-
-Improve run summary output:
-
-- pipeline id;
-- source backend;
-- discovered bundle count;
-- destination ids;
-- action per bundle/destination;
-- final status.
-
-### Tests
-
-Add tests for:
-
-- notifier is invoked at the expected orchestration point where testable;
-- pipeline failure when a destination fails;
-- run summary contains meaningful status information;
-- dry-run does not invoke write-side effects.
-
-### Completion Criteria
-
-- The pipeline shape includes notification as an internal no-op stage.
-- Run output is useful for unattended operation logs.
-- Local MVP behavior remains passing and is ready for one real local producer pipeline.
-
-Stages 1 through 8 define the local MVP checkpoint. Later stages extend the local MVP with remote backends, cross-backend hardening, user-facing documentation sync, and release readiness.
-
-## Stage 9: Native SSH/SFTP Backend Roadmap Extension
-
-### Goal
-
-Implement SSH/SFTP storage backend support for sources and destinations.
-
-### Scope
-
-Create:
-
-```text
-internal/adapters/ssh/backend.go
-internal/adapters/ssh/config.go
-```
-
-Implement the storage backend interface over native SSH/SFTP.
-
-Required config:
-
-```yaml
-backend: ssh
-uri: ssh://user@example.com:22
-path: /remote/root
-```
-
-Authentication expectations:
-
-- prefer SSH agent by default;
-- use known_hosts validation by default where practical;
-- do not require passwords in YAML;
-- optional key-file support may be implemented if straightforward, but should not distract from agent-based auth.
-
-Support SSH/SFTP backend as both source and destination:
-
-- local -> ssh;
-- ssh -> local;
-- ssh -> ssh where feasible through staging or streaming.
-
-### Safety Requirements
-
-The SSH backend must enforce the same logical path safety rules as the local backend.
-
-Deletion must remain limited to managed destination bundle paths guarded by valid `.distributor.json`.
-
-### Tests
-
-Unit-test path handling and config validation.
-
-If practical, add integration tests that can be skipped unless an SSH test endpoint is configured through environment variables. Do not require a live SSH server for normal `go test ./...`.
-
-### Completion Criteria
-
-- SSH/SFTP backend compiles and satisfies the storage interface.
-- Backend config validation is tested.
-- Normal tests do not depend on a live SSH server.
-- At least local-to-SSH and SSH-to-local flows are documented or manually testable.
-
-## Stage 10: S3-Compatible Backend Roadmap Extension
-
-### Goal
-
-Implement S3-compatible backend support for sources and destinations.
-
-### Scope
-
-Create:
-
-```text
-internal/adapters/s3/backend.go
-internal/adapters/s3/config.go
-```
-
-Required config should align with `docs/roadmap/config.md`:
-
-```yaml
-backend: s3
-endpoint: https://s3.example.com
-bucket: reports
-prefix: some/prefix
-region: us-east-1
-force_path_style: true
-credentials:
-  access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
-  secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
-```
-
-Implement storage operations over S3 object keys through the common storage contract:
-
-- `ReadFile` and `OpenReader`;
-- `WriteFile` and `WriteFrom`;
-- `Stat`;
-- `Walk` using object-list pagination where available;
-- `HasAny` with early stop for destination emptiness checks;
-- `DeleteManagedBundle`;
-- read/write `.distributor.json` through normal storage operations.
-
-Set reasonable content types where available:
-
-- `.md`: `text/markdown; charset=utf-8`;
-- `.html`: `text/html; charset=utf-8`;
-- `.json`: `application/json`;
-- `.txt`: `text/plain; charset=utf-8`.
-
-Support S3 backend as both source and destination.
-
-### Safety Requirements
-
-Treat S3 prefixes as object trees. Do not assume real directories exist.
-
-Deletion must be limited to destination bundle prefixes that are confirmed managed by `.distributor.json`.
-
-### Tests
-
-Add unit tests for:
-
-- config validation;
-- key/prefix normalization;
-- content type selection;
-- path traversal rejection;
-- publish planning with S3 destination state fixtures.
-
-If practical, add integration tests gated by environment variables or a local S3-compatible test service. Normal `go test ./...` must not require live S3 credentials.
-
-### Completion Criteria
-
-- S3 backend compiles and satisfies the storage interface.
-- S3 source and destination flows are supported through the common pipeline path.
-- Normal tests do not require live S3.
-
-## Stage 11: Cross-Backend End-to-End Coverage and Hardening Roadmap Extension
-
-### Goal
-
-Harden the MVP across backend combinations, destination policies, and failure cases.
-
-### Scope
-
-Add end-to-end coverage for representative scenarios:
-
-- local source -> local archive destination;
-- local source -> local HTML destination;
-- local source -> two destinations with different publish policies;
-- local source -> SSH destination, where integration credentials exist;
-- local source -> S3 destination, where integration credentials exist;
-- S3 source -> local destination, where integration credentials exist;
-- SSH source -> local destination, where integration credentials exist.
-
-Improve logging and error messages for:
-
-- invalid config;
-- invalid source manifest;
-- digest mismatch;
-- destination conflict;
-- unmanaged destination path;
-- backend read/write/list failures;
-- transform failures.
-
-Ensure all destructive paths have tests or explicit safeguards.
-
-### Tests
-
-Add or expand tests for:
-
-- dry-run across multiple destinations;
-- partial failure behavior;
-- repeated run idempotency;
-- older/newer destination state behavior;
-- destination state output metadata accuracy;
-- generated HTML output metadata accuracy.
-
-### Completion Criteria
-
-- MVP behavior is reliable across implemented backend types.
-- Error messages identify pipeline id, destination id, bundle id, and reason where practical.
-- Idempotent repeated runs behave as expected.
-
-## Stage 12: User-Facing Documentation Sync
-
-### Goal
-
-Update documentation to reflect implemented MVP behavior.
-
-### Scope
-
-Following `docs/policy/documentation.md`, create or update user-facing documentation only for implemented features.
-
-Likely docs:
-
-```text
-README.md
-docs/config.md
-docs/cli.md
-docs/policy/architecture.md
-docs/internal/bundles.md
-docs/internal/backends.md
-```
-
-Document:
-
-- what `distributor` does;
-- bundle contract summary;
-- `.distributor.json` role;
-- example source bundle;
-- example local-to-local config;
-- example local-to-S3 config;
-- example local-to-SSH config;
-- `run`, `validate`, and `inspect` commands;
-- dry-run behavior;
-- replacement and safety rules;
-- Markdown-to-HTML transform behavior;
-- environment-variable credential handling.
-
-Move roadmap material to historical/planning status only if your documentation policy allows it. Do not describe unimplemented notification adapters as available features.
-
-### Tests
-
-Run the full test suite.
-
-If docs include command examples, verify that basic examples correspond to actual CLI behavior.
-
-### Completion Criteria
-
-- User-facing docs describe the implemented MVP accurately.
-- Roadmap docs no longer masquerade as implemented behavior.
-- `go test ./...` passes.
-
-## Stage 13: MVP Release Readiness Pass
-
-### Goal
-
-Perform a final pre-release quality pass.
-
-### Scope
-
-Review:
-
-- package boundaries against `docs/policy/architecture.md`;
-- package layout against `docs/roadmap/packages.md`;
-- implemented contracts against `docs/roadmap/contracts.md`;
-- implemented config behavior against `docs/roadmap/config.md`;
-- docs against `docs/policy/documentation.md`;
-- destructive operation safety;
-- logs and errors for unattended operation;
-- command UX;
-- test coverage for core invariants.
-
-Add any missing small tests or docs discovered during review.
-
-Do not add new product features in this stage.
-
-### Completion Criteria
-
-- MVP is ready to deploy against one real producer pipeline.
-- A dry-run can be performed safely against a real source and destination.
-- Repeated runs are idempotent.
-- Destructive replacement cannot occur outside managed destination bundle paths.
-- Final docs accurately reflect the application.
-
-## Stage 14: Explicit Force Overwrite Roadmap Extension
-
-### Goal
-
-Introduce explicit operator-requested force behavior for controlled overwrite cases that are intentionally outside the local MVP.
-
-### Scope
-
-Add a CLI-only force option such as:
-
-```bash
-distributor run --config config.yml --force
-```
-
-Define and implement force planning for:
-
-- unmanaged non-empty destination paths;
-- destination state with a different source id;
-- destination state with matching source id and created timestamp but different digest;
-- destination state with mismatched `pipeline_id` or `destination_id`;
-- destination newer than source when the transfer policy explicitly allows replacement.
-
-Force behavior must be explicit per run. It should not be a persistent default in config for this stage.
-
-Update transfer policy validation to allow broader values only when force behavior is implemented and documented:
-
-- `on_destination_newer: replace`
-- `on_conflict: replace`
-
-### Safety Requirements
-
-- Dry-run must show every file or object that would be written or deleted before a forced run.
-- Force must still never delete above the resolved destination bundle path or configured destination prefix.
-- Filesystem replacement should remain staged where practical.
-- S3 replacement must remain constrained to the destination bundle prefix.
-- Logs must clearly mark force decisions and include pipeline id, destination id, bundle id, and reason.
-
-### Tests
-
-Add tests for:
-
-- force rejected when the flag is absent;
-- unmanaged non-empty destination overwritten only with force;
-- different source id overwritten only with force and allowed policy;
-- same id and created but different digest overwritten only with force and allowed policy;
-- destination newer replaced only with force and allowed policy;
-- pipeline or destination id mismatch overwritten only with force and allowed policy;
-- dry-run reports destructive force actions without writing;
-- destructive paths remain bounded to the destination bundle path.
-
-### Completion Criteria
-
-- Force overwrite behavior is explicit, logged, dry-runnable, and test-covered.
-- Default non-force behavior remains unchanged and conservative.
-
-## Deferred Post-MVP Work
-
-The following items are intentionally outside the MVP unless explicitly pulled into a later roadmap:
-
-- email notifications;
-- ntfy/Gotify/Pushover notifications;
-- RSS/Atom feed generation;
-- static site index pages beyond sidecar HTML output;
-- templated HTML themes beyond a minimal deterministic template;
-- destination path remapping rules;
-- full plugin architecture;
-- web UI;
-- report editing;
-- producer pipeline execution;
-- database-backed state;
-- complex retry queues;
-- concurrent publication workers;
-- symlink support;
-- warning-only digest mismatch handling;
-- password-based SSH authentication in YAML.

docs/roadmap/packages.md

@@ -1,526 +0,0 @@
-# Package Layout Roadmap
-
-This roadmap defines the proposed package layout, boundaries, and implementation responsibilities for the `distributor` MVP.
-
-`distributor` is expected to be a domain-agnostic bundle publisher. Producer applications emit source bundles containing `manifest.json`; `distributor` validates those bundles and publishes selected source and generated artifacts to one or more configured destinations.
-
-This document is roadmap material. It describes the intended package design before implementation and should move into `docs/internal/` only after corresponding behavior exists.
-
-## Accepted Package Layout
-
-```text
-cmd/distributor/
-  main.go
-
-internal/app/
-  app.go
-  run.go
-  pipeline.go
-
-internal/cli/
-  root.go
-  run.go
-  validate.go
-  inspect.go
-
-internal/config/
-  config.go
-  defaults.go
-  load.go
-  validate.go
-
-internal/bundle/
-  manifest.go
-  digest.go
-  validate.go
-  discover.go
-
-internal/state/
-  distributor.go
-  compare.go
-  validate.go
-
-internal/storage/
-  backend.go
-  registry.go
-  path.go
-  errors.go
-
-internal/storage/fake/
-  backend.go
-
-internal/adapters/local/
-  backend.go
-
-internal/adapters/ssh/
-  backend.go
-  config.go
-
-internal/adapters/s3/
-  backend.go
-  config.go
-
-internal/transform/
-  transform.go
-  registry.go
-  plan.go
-
-internal/transform/markdown/
-  markdown.go
-  template.go
-
-internal/publish/
-  plan.go
-  execute.go
-  output.go
-  reconcile.go
-  safety.go
-
-internal/notify/
-  notify.go
-  noop.go
-
-internal/logging/
-  logging.go
-```
-
-## Package Responsibilities
-
-### `cmd/distributor`
-
-Application entrypoint only.
-
-Responsibilities:
-
-- call CLI execution;
-- translate process exit status;
-- avoid business logic.
-
-Non-responsibilities:
-
-- config loading;
-- backend construction;
-- bundle validation;
-- publish decisions.
-
-### `internal/cli`
-
-CLI command definitions, flags, argument parsing, and command wiring.
-
-Expected MVP commands:
-
-- `distributor run` — run configured pipelines.
-- `distributor run --dry-run` — plan without modifying destinations.
-- `distributor run --pipeline <id>` — run one configured pipeline.
-- `distributor validate <path>` — validate a source bundle or source tree where feasible.
-- `distributor inspect <path>` — inspect a bundle or destination state where feasible.
-
-Boundaries:
-
-- CLI should call `internal/app` use cases.
-- CLI should not parse manifests directly except through application APIs.
-- CLI should not import backend adapter implementation details unless only for registration side effects.
-
-### `internal/app`
-
-Application orchestration and top-level use cases.
-
-Responsibilities:
-
-- load and validate configuration;
-- construct configured pipelines;
-- build source and destination backends through registries;
-- orchestrate discovery, validation, planning, publishing, and notification;
-- coordinate dry-run output;
-- run destination fan-out deterministically and sequentially;
-- aggregate destination outcomes into run-level failure behavior.
-
-Core orchestration shape:
-
-```text
-for each selected pipeline:
-  open source backend
-  discover source bundles
-  for each source bundle:
-    validate source manifest and digest
-    for each destination:
-      inspect .distributor.json
-      build publish plan
-      transform as required by that destination
-      execute publish plan unless dry-run
-      run noop notifier after actual publication or replacement
-```
-
-Boundaries:
-
-- `internal/app` composes packages but should not contain backend-specific logic.
-- Publish decisions should live in `internal/publish`, not inline in orchestration.
-- Destination state comparison should live in `internal/state` or `internal/publish`, not CLI code.
-
-### `internal/config`
-
-Configuration structs, defaults, loading, precedence, and validation.
-
-Responsibilities:
-
-- load `/usr/local/etc/distributor/config.yml` by default;
-- support `--config` override;
-- apply defaults;
-- validate required fields;
-- validate pipeline ids and destination ids;
-- validate backend-specific config shapes;
-- validate transform and publish policy combinations.
-
-MVP config model:
-
-```yaml
-pipelines:
-  - id: weather-daily
-    source:
-      backend: local
-      path: /var/spool/distributor/weather
-    validation:
-      on_digest_mismatch: fail
-    destinations:
-      - id: markdown-archive
-        backend: s3
-        endpoint: https://s3.example.com
-        bucket: reports
-        prefix: weather/archive
-        region: us-east-1
-        force_path_style: true
-        publish:
-          source: true
-          html: false
-        transfer:
-          on_destination_same: skip
-          on_destination_older: replace
-          on_destination_newer: skip
-          on_conflict: fail
-      - id: static-site
-        backend: ssh
-        uri: ssh://deploy@example.com:22
-        path: /srv/www/weather
-        publish:
-          source: false
-          html: true
-        transform:
-          markdown_to_html:
-            enabled: true
-            mode: sidecar
-```
-
-Configuration principles:
-
-- one source per pipeline;
-- one or more destinations per pipeline;
-- transforms are destination-specific;
-- publish policy is destination-specific;
-- secrets should use environment variables, secret files, SSH agent, or standard credential mechanisms rather than raw YAML values.
-
-### `internal/bundle`
-
-Source bundle contract and validation.
-
-Responsibilities:
-
-- parse source `manifest.json`;
-- represent source manifests and files;
-- discover bundle roots beneath a configured source root;
-- validate required manifest fields;
-- validate RFC3339 `created` values;
-- validate relative paths;
-- validate file existence, size, per-file SHA-256, and bundle digest;
-- expose normalized source bundle models to other packages.
-
-Core types:
-
-```go
-type Manifest struct {
-    SchemaVersion int
-    ID      string
-    Digest  string
-    Created time.Time
-    Files   []ManifestFile
-}
-
-type ManifestFile struct {
-    Path   string
-    SHA256 string
-    Size   int64
-}
-
-type Bundle struct {
-    RootRelativePath string
-    Manifest         Manifest
-}
-```
-
-Boundaries:
-
-- `internal/bundle` does not know about `.distributor.json`.
-- `internal/bundle` does not know about destinations, transforms, or notification.
-- `internal/bundle` may use the storage abstraction to read source files, but it should not import backend adapter packages.
-
-### `internal/state`
-
-Destination state contract for `.distributor.json`.
-
-Responsibilities:
-
-- parse `.distributor.json`;
-- validate destination state;
-- represent copied source outputs and generated outputs;
-- embed the source manifest used for publication;
-- compare destination state against a current source manifest;
-- classify destination state as same, older, newer, conflict, absent, invalid, or unmanaged.
-
-Core types:
-
-```go
-type DistributorState struct {
-    SchemaVersion int
-    DistributorVersion string
-    PipelineID    string
-    DestinationID string
-    PublishedAt   time.Time
-    Source        SourceState
-    Outputs       []OutputFile
-}
-
-type SourceState struct {
-    Manifest bundle.Manifest
-}
-
-type OutputFile struct {
-    Path       string
-    Kind       string // source | generated
-    SourcePath string
-    Transform  string
-    SHA256     string
-    Size       int64
-}
-```
-
-Comparison rules:
-
-- same source manifest: skip;
-- same source id, older destination source `created`: replace;
-- same source id, newer destination source `created`: skip;
-- same source id, same `created`, different digest: conflict;
-- different source id: conflict;
-- pipeline id or destination id mismatch: conflict;
-- absent state: publish only if safe;
-- unmanaged non-empty path: fail.
-
-Boundaries:
-
-- `internal/state` owns destination state semantics, not publish execution.
-- `internal/state` should not know about S3, SSH/SFTP, local filesystem details, or Markdown rendering.
-
-### `internal/storage`
-
-Backend abstraction and shared storage types.
-
-Responsibilities:
-
-- define storage backend interfaces;
-- define object/file metadata types;
-- define path/prefix helpers;
-- define common storage errors;
-- provide backend registry mechanisms;
-- provide a fake backend for core package tests.
-
-The detailed storage contract is defined in `docs/roadmap/storage.md`. Core application code should use that storage interface for backend-rooted logical paths, byte and stream IO, metadata, traversal, typed errors, emptiness checks, and managed deletion.
-
-Destructive APIs should remain narrow. Prefer managed deletion of files recorded in `.distributor.json` instead of broad recursive deletion.
-
-Boundaries:
-
-- `internal/storage` should not contain backend implementation details.
-- Adapter dependencies must not leak through storage interfaces.
-- The fake backend exists for tests and should not become an application runtime backend.
-
-### `internal/adapters/local`
-
-Local filesystem backend.
-
-Responsibilities:
-
-- implement `storage.Backend` for local paths;
-- clean and constrain paths;
-- perform safe reads/writes/listing/deletion;
-- use atomic writes where practical;
-- reject unsafe path traversal;
-- handle symlink policy explicitly.
-
-Testing expectations:
-
-- use temporary directories;
-- verify path traversal rejection;
-- verify write and delete safety.
-
-### `internal/adapters/ssh`
-
-SSH/SFTP backend.
-
-Responsibilities:
-
-- implement `storage.Backend` over SSH/SFTP;
-- support `uri` and `path` config;
-- prefer native SFTP implementation;
-- use SSH agent, key files, known hosts, or documented auth mechanisms;
-- avoid raw passwords in config unless explicitly designed and documented later;
-- translate SSH/SFTP errors into storage-level errors.
-
-Testing expectations:
-
-- core app tests should use fake backends;
-- adapter tests may use local test servers or targeted integration tests if practical;
-- do not require a real production SSH host for normal unit tests.
-
-### `internal/adapters/s3`
-
-S3-compatible object storage backend.
-
-Responsibilities:
-
-- implement `storage.Backend` over S3-compatible object storage;
-- support endpoint, bucket, prefix, region, and force-path-style configuration;
-- support standard credential mechanisms or explicit environment-variable references;
-- treat S3 as an object tree, not a filesystem;
-- set reasonable content types where practical;
-- guard against prefix/root deletion mistakes.
-
-Testing expectations:
-
-- core app tests should use fake backends;
-- adapter behavior may be tested through mocks, local S3-compatible services, or narrow integration tests;
-- config examples should avoid real secrets.
-
-### `internal/transform`
-
-Transform interfaces, registry, and transform planning.
-
-Responsibilities:
-
-- define transform interfaces;
-- register available transforms;
-- represent transform requests and outputs;
-- keep transform execution independent of destination backend details.
-
-Boundaries:
-
-- transforms operate on source bundle content and destination transform config;
-- transforms do not publish outputs;
-- transforms do not mutate source bundles;
-- transforms should return generated output metadata for `.distributor.json`.
-
-### `internal/transform/markdown`
-
-Markdown-to-HTML implementation.
-
-Responsibilities:
-
-- render listed Markdown files to HTML;
-- support MVP sidecar behavior, such as `report.md` -> `report.html`;
-- record generated output path, source path, transform name, SHA-256, and size;
-- optionally use embedded templates if needed.
-
-MVP scope:
-
-- Markdown to HTML only;
-- no PDF generation;
-- no email-specific HTML;
-- no complex theming unless required for basic output correctness.
-
-### `internal/publish`
-
-Destination planning, reconciliation, safety checks, and publish execution.
-
-Responsibilities:
-
-- inspect destination state;
-- plan destination action;
-- enforce destination conflict rules;
-- enforce destructive-operation safety rules;
-- detect output path collisions before writing;
-- combine source files and transform outputs according to destination publish policy;
-- write destination outputs;
-- write `.distributor.json`;
-- use staging or equivalent cleanup behavior where practical;
-- support dry-run planning;
-- report skipped, replaced, failed, and published actions.
-
-Action model:
-
-```text
-publish
-replace
-skip_same
-skip_destination_newer
-fail_conflict
-fail_unmanaged
-```
-
-Boundaries:
-
-- publish logic should not parse CLI flags;
-- publish logic should not know adapter implementation details;
-- publish logic should use `internal/state` for destination state semantics;
-- publish logic should use `internal/storage` interfaces for IO.
-
-### `internal/notify`
-
-Notification stage abstraction.
-
-MVP responsibilities:
-
-- define notifier interface;
-- implement no-op notifier;
-- preserve future extension point for email, ntfy, Gotify, RSS update hooks, or other notification channels.
-
-Future notification rules:
-
-- notify only after successful publication to the relevant destination or destinations;
-- notification must be idempotent with respect to source id, digest, pipeline id, and destination id where applicable;
-- notification should not run for skipped or failed publications unless explicitly configured.
-
-### `internal/logging`
-
-Logging setup and helpers.
-
-Responsibilities:
-
-- centralize structured logging setup;
-- ensure logs omit secrets;
-- provide consistent fields for pipeline id, bundle id, destination id, backend, path, action, and reason.
-
-## Deferred Ideas
-
-The following are intentionally out of MVP unless separately accepted in a later roadmap:
-
-- email, ntfy, Gotify, or other real notification adapters;
-- RSS/Atom feed generation;
-- PDF generation;
-- web UI;
-- full-text search;
-- dynamic plugin loading;
-- arbitrary transform chains;
-- workflow DAGs;
-- producer execution;
-- complex templating/theming;
-- bidirectional sync;
-- backup semantics.
-
-## Key Invariants
-
-- Producer apps own source bundle creation.
-- `distributor` owns destination publication state.
-- Source `manifest.json` is not copied as destination state.
-- Destination `.distributor.json` is the managed sentinel.
-- One pipeline has one source and one or more destinations.
-- Transform and publish policy are destination-specific.
-- Source files are canonical; HTML is derived.
-- Destructive replacement is allowed only inside managed destination bundle paths.
-- Core logic must be testable without real S3, SSH, or remote services.

docs/roadmap/storage.md

@@ -1,265 +0,0 @@
-# Storage Interface Roadmap
-
-This roadmap defines the planned `internal/storage` contract for the `distributor` MVP. The goal is to give bundle validation, destination state inspection, publish planning, and backend adapters one consistent IO boundary without leaking local filesystem, SSH/SFTP, or S3-specific behavior into core packages.
-
-## Purpose and Invariants
-
-The storage layer is responsible for safe, backend-rooted access to files, objects, prefixes, and destination bundle paths.
-
-Core invariants:
-
-- Backends are opened at configured roots.
-- Core packages operate on backend-rooted logical paths, not absolute filesystem paths or raw object keys.
-- Backend adapters translate native storage behavior into common storage entries and typed errors.
-- Destructive operations remain narrow and managed.
-- Staging or atomic write behavior belongs behind the storage interface where practical.
-- The fake backend exists for tests only and must not be registered as a runtime backend.
-
-## Logical Path Model
-
-Storage paths are slash-separated logical paths relative to an already configured backend root.
-
-File paths:
-
-- must be non-empty;
-- must be relative;
-- must be clean;
-- must not contain `.` or `..` segments;
-- must not start with `/`;
-- must not contain backslashes;
-- must not resolve outside the backend root.
-
-Prefix paths use the same slash-separated model. A prefix may be empty to represent the backend root for traversal and destination emptiness checks.
-
-Prefix matching must preserve logical path boundaries. A prefix of `foo` matches `foo` and entries below `foo/`; it must not match a sibling path such as `foobar`. Backends that map logical paths to object keys must apply the same normalized boundary rule after combining configured backend prefixes with caller-provided logical prefixes.
-
-Backends own conversion from logical paths to native paths or object keys. Core packages should not construct local filesystem paths, SFTP paths, or S3 object keys directly.
-
-## Core Interface Shape
-
-The MVP should use a hybrid byte and stream interface:
-
-```go
-type Backend interface {
-    ReadFile(ctx context.Context, path string) ([]byte, error)
-    OpenReader(ctx context.Context, path string) (io.ReadCloser, error)
-    WriteFile(ctx context.Context, path string, data []byte, opts WriteOptions) (Entry, error)
-    WriteFrom(ctx context.Context, path string, r io.Reader, opts WriteOptions) (Entry, error)
-    Stat(ctx context.Context, path string) (Entry, error)
-    Walk(ctx context.Context, prefix string, opts WalkOptions, fn WalkFunc) error
-    HasAny(ctx context.Context, prefix string) (bool, error)
-    DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts DeleteOptions) error
-}
-```
-
-Byte helpers are expected to cover manifests, destination state, small source artifacts, and generated outputs. Stream methods are included from the start for backend flexibility and larger future artifacts.
-
-Write operations should create required parent directories or prefixes as needed.
-
-Concrete option and callback types should use this shape:
-
-```go
-type WalkOptions struct {
-    Recursive bool
-    Limit     int
-}
-
-type WalkFunc func(Entry) error
-
-var ErrStopWalk = errors.New("stop walk")
-
-type WriteOptions struct {
-    ContentType  string
-    Overwrite    bool
-    PreferAtomic bool
-    Size         int64
-    SizeKnown    bool
-}
-
-type DeleteOptions struct {
-    IgnoreMissing  bool
-    PruneEmptyDirs bool
-}
-```
-
-`WalkOptions.Limit == 0` means no explicit limit. `SizeKnown` applies primarily to `WriteFrom`; byte writes can infer size from the provided data.
-
-## Entries and Metadata
-
-Storage metadata should be represented by an `Entry` model with at least:
-
-- backend-relative logical path;
-- entry type;
-- size, where available.
-
-Entry types:
-
-- `file`: filesystem file or object-storage object;
-- `directory`: filesystem directory or logical prefix;
-- `symlink`: local filesystem symlink;
-- `other`: unknown or unsupported native entry type.
-
-`Stat` returns metadata for one exact logical path. It may report a real filesystem directory, symlink, file, or exact object. It must not synthesize S3-like directory metadata solely because objects exist below a prefix; callers that need prefix existence or destination emptiness must use `HasAny` or `Walk`.
-
-`Walk` traverses entries below a prefix and calls a callback for each entry. `WalkOptions` should include:
-
-- whether traversal is recursive;
-- an optional entry limit for callers that only need to know whether content exists.
-
-If a callback returns `ErrStopWalk`, traversal stops successfully and `Walk` returns nil. Any other callback error stops traversal and is returned with storage context where practical. If `WalkOptions.Limit` is greater than zero, reaching the limit stops traversal successfully.
-
-Backends may stream or paginate traversal internally. S3-compatible adapters should not need to load a whole prefix into memory to satisfy traversal.
-
-Raw traversal is not required to be lexically sorted. A helper that materializes walk results for bundle discovery, tests, or CLI output should sort entries lexically by logical path before returning them.
-
-`HasAny` reports whether at least one entry exists below a prefix. It should stop as soon as content is found.
-
-Source validation must reject symlink entries reported by local `Stat` or `Walk`.
-
-## Read Behavior
-
-`ReadFile` reads the whole object into memory and is appropriate for MVP manifest, state, and ordinary artifact handling.
-
-`OpenReader` returns a stream for callers that need to copy or hash content without requiring a second storage-specific API. Callers must close the returned reader.
-
-Both read methods must:
-
-- validate logical paths before backend access;
-- reject directories, prefixes, symlinks, and unsupported entries;
-- return typed not-found and invalid-path errors where applicable.
-
-## Write Behavior
-
-`WriteOptions` should include:
-
-- content type, when the destination backend can use it;
-- overwrite permission;
-- atomic or staged write preference;
-- optional known size for stream writes.
-
-Backends own staging and atomic behavior where practical:
-
-- Local backend writes to a temporary file in the destination directory and renames or promotes into place.
-- SSH/SFTP backend should use a temporary remote file and rename where available.
-- S3-compatible backend treats a successful object PUT as publish-on-success and applies content type metadata.
-
-Remote adapters may buffer or spool `WriteFrom` input when needed to satisfy backend requirements such as content length, multipart upload, or retry behavior. Callers that know the stream size should set `SizeKnown` and `Size`.
-
-If overwrite is false and the target exists, writes should fail with an already-exists error.
-
-`WriteFile` and `WriteFrom` should return the written `Entry`, including final path and size where available.
-
-`DeleteOptions` should include:
-
-- whether missing managed output paths are ignored;
-- whether empty parent directories may be pruned for filesystem-like backends.
-
-## Managed Deletion
-
-The storage interface should expose a guarded managed deletion operation rather than raw recursive delete.
-
-`DeleteManagedBundle(ctx, bundlePath, managedOutputPaths, opts)` may delete only:
-
-- files or objects listed in valid `.distributor.json.outputs`;
-- `.distributor.json` at the destination bundle path;
-- empty directories created by those files, for filesystem-like backends.
-
-`managedOutputPaths` are relative to the destination bundle path. The backend validates each path and resolves it under `bundlePath`.
-
-If `bundlePath == ""`, deletion may remove explicit managed files at the destination root, but must never delete the root itself.
-
-Prefix or recursive deletion is out of MVP scope. A future force-overwrite stage may add broader behavior, but it must remain explicit and separately documented.
-
-## Destination Emptiness
-
-Destination emptiness should use `HasAny(prefix)` and typed not-found behavior. Callers that only need emptiness must not materialize a full recursive traversal.
-
-Rules:
-
-- A local destination bundle path is empty when the directory does not exist or exists with no entries.
-- An S3-compatible prefix is empty when no objects exist below that exact destination bundle prefix.
-- Entries outside the exact destination bundle path or prefix do not affect emptiness.
-
-## Error Model
-
-Storage should expose typed error categories with wrapping context. Callers should use helper predicates rather than string matching.
-
-Required categories:
-
-- not found;
-- already exists;
-- not empty;
-- invalid path;
-- conflict;
-- permission;
-- temporary;
-- unsupported;
-- unknown.
-
-Adapters should translate backend-native errors into these categories while preserving useful operation, backend, path, and cause context.
-
-## Adapter Expectations
-
-### Local
-
-The local backend should:
-
-- constrain all operations beneath the configured root;
-- reject traversal and absolute logical paths;
-- report symlinks through metadata;
-- reject symlink reads for source artifacts;
-- use staged writes where practical;
-- perform managed deletion only for explicit managed files and `.distributor.json`;
-- clean up empty directories created by managed outputs where safe.
-
-### Fake
-
-The fake backend should:
-
-- be in-memory and deterministic;
-- implement the same logical path validation rules;
-- support `Stat`, `Walk`, `HasAny`, byte reads and writes, stream reads and writes, managed deletion, and destination emptiness helper behavior;
-- support configured symlink entries for validation tests;
-- be used only by tests.
-
-### SSH/SFTP
-
-The SSH/SFTP backend should:
-
-- use native SFTP operations;
-- enforce the same logical path rules as local storage;
-- use temporary file plus rename for staged writes where available;
-- translate remote errors into storage error categories;
-- avoid exposing SSH or SFTP dependency types through `internal/storage`.
-
-### S3-Compatible
-
-The S3-compatible backend should:
-
-- treat prefixes as object trees, not real directories;
-- normalize configured prefix plus logical path into object keys;
-- use object PUT as publish-on-success;
-- set content type from `WriteOptions`;
-- implement traversal and emptiness by exact prefix;
-- use backend pagination for traversal where available;
-- allow `HasAny` to stop after the first matching object;
-- constrain managed deletion to listed output objects and `.distributor.json`.
-
-## Tests and Fixtures
-
-Storage implementation stages should test:
-
-- path validation rejects absolute paths, traversal, empty file paths, backslashes, and dot segments;
-- `Walk` visits backend-rooted logical paths under a prefix and supports recursive traversal;
-- a materializing helper sorts walk results lexically for deterministic tests and CLI output;
-- `HasAny` returns quickly for non-empty prefixes without requiring full traversal;
-- `ReadFile` and `OpenReader` return equivalent bytes;
-- `WriteFile` and `WriteFrom` honor overwrite and content-type options;
-- local staged writes do not leave final files on failure where testable;
-- managed deletion deletes only state-listed files and `.distributor.json`;
-- managed deletion never deletes destination root or unlisted files;
-- destination emptiness helper handles missing, empty, and non-empty local paths;
-- S3-compatible traversal can use pagination without loading a whole prefix into memory;
-- symlink entries are reported and rejected by source validation;
-- typed errors are usable through helper predicates;
-- fake backend behavior matches local backend semantics relevant to core tests.

docs/troubleshooting.md

@@ -1,8 +1,14 @@
 # Distributor Troubleshooting
 
-## `load config ... no such file or directory`
+Audience: administrators and operators diagnosing `distributor` command, configuration, publishing, storage, or HTTP upload failures.
 
-Likely cause: `run` could not find the config path. If `--config` is omitted, the default path is `/usr/local/etc/distributor/config.yml`.
+Each entry lists the symptom, likely cause, diagnostic step, safe fix, and relevant reference link. Command syntax lives in [CLI](cli.md), configuration fields live in [Configuration](config.md), and operating procedures live in [Operations](operations.md).
+
+## Config File Is Missing
+
+Symptom: `load config ... no such file or directory`.
+
+Likely cause: `--config` points to a missing file, or `--config` was omitted and `/usr/local/etc/distributor/config.yml` is not installed.
 
 Diagnostic:
 
@@ -10,11 +16,15 @@ Diagnostic:
 ls -l <config-path>
 ```
 
-Safe fix: pass an existing config path with `--config`, or install a config at the default path. See [configuration](config.md).
+Safe fix: pass an existing file with `--config`, or install a config at the default path.
 
-## `parse config ... field not found`
+Reference: [Configuration](config.md#config-file-loading).
 
-Likely cause: the YAML contains an unknown field. Config loading rejects unknown keys.
+## Config Contains An Unknown Field
+
+Symptom: `parse config ... field not found`.
+
+Likely cause: the YAML contains a key that is not part of the implemented config schema.
 
 Diagnostic:
 
@@ -22,11 +32,15 @@ Diagnostic:
 go run ./cmd/distributor run --config <config-path> --dry-run
 ```
 
-Safe fix: compare the file to the reference in [configuration](config.md) and remove or rename unsupported fields.
+Safe fix: remove or rename unsupported fields using the canonical config reference.
 
-## `validate config ... backend ... is unsupported`
+Reference: [Configuration](config.md).
 
-Likely cause: a source or destination uses a backend name other than `local`, `ssh`, or `s3`.
+## Backend Name Or Placement Is Invalid
+
+Symptom: `backend ... is unsupported` or `http_upload is only supported for sources`.
+
+Likely cause: a backend name is misspelled, not executable, or configured in the wrong role.
 
 Diagnostic:
 
@@ -34,23 +48,15 @@ Diagnostic:
 rg -n "backend:" <config-path>
 ```
 
-Safe fix: use `backend: local` for executable workflows. SSH and S3 config shapes are accepted only for validation; runtime execution is unavailable.
+Safe fix: use `local`, `ssh`, or `s3` for executable sources and destinations. Use `http_upload` only as a source served by `distributor serve`.
 
-## `backend ssh is not implemented for execution` or `backend s3 is not implemented for execution`
+Reference: [Configuration](config.md#backend-reference).
 
-Likely cause: the config validates but `run` tried to execute a remote backend.
+## CLI Arguments Select The Wrong Source Mode
 
-Diagnostic:
+Symptom: `configured source mode requires --pipeline`, `does not accept a local path with --config, --pipeline, or --bundle`, `validate command requires a path`, or `inspect command requires a path`.
 
-```sh
-go run ./cmd/distributor run --config <config-path> --dry-run
-```
-
-Safe fix: use local destinations for current executable workflows, or keep remote backend configs under roadmap material unless those adapters are added. See [configuration](config.md).
-
-## `validate command requires a path` or `inspect command requires a path`
-
-Likely cause: `validate` or `inspect` was run without a path.
+Likely cause: `validate` or `inspect` mixed local path mode with configured source mode, or omitted the required source selector.
 
 Diagnostic:
 
@@ -59,23 +65,79 @@ go run ./cmd/distributor validate --help
 go run ./cmd/distributor inspect --help
 ```
 
-Safe fix: pass a local source bundle directory or a local tree containing source bundles.
+Safe fix: use either `distributor validate <path>` / `distributor inspect <path>`, or use `--config <path> --pipeline <id>` with optional `--bundle <path>`.
 
-## `no bundles found under "."`
+Reference: [CLI](cli.md#validate).
 
-Likely cause: the selected source root does not contain a `manifest.json` source bundle.
+## Output Format Is Invalid
+
+Symptom: `format must be text or json`.
+
+Likely cause: an unsupported value was passed to `--format`.
 
 Diagnostic:
 
 ```sh
-find <source-root> -name manifest.json -print
+go run ./cmd/distributor run --help
 ```
 
-Safe fix: point the command or config at the directory containing the source bundle, or write a valid `manifest.json` and listed files. See [CLI](cli.md).
+Safe fix: use `--format text` or `--format json`.
 
-## `sha256 mismatch`, `size mismatch`, or `digest mismatch`
+Reference: [CLI](cli.md#common-output-format).
 
-Likely cause: a listed source file changed after `manifest.json` was created, or the manifest digest does not match its file list.
+## JSON Mode Wrote No JSON Document
+
+Symptom: `--format json` exits non-zero and stdout has no JSON result.
+
+Likely cause: the command failed before it could construct a result, such as invalid arguments, missing config, unreadable secrets, or source setup failure.
+
+Diagnostic:
+
+```sh
+go run ./cmd/distributor run --config <config-path> --format json
+```
+
+Safe fix: read stderr, fix the setup problem, then rerun. Partial destination failures during `run` can produce JSON; fatal setup failures do not.
+
+Reference: [CLI](cli.md#output-and-exit-behavior).
+
+## Source Pipeline Is Not Found
+
+Symptom: `pipeline "<id>" not found`.
+
+Likely cause: configured source diagnostics or upload processing selected a pipeline id that is absent from the loaded config.
+
+Diagnostic:
+
+```sh
+rg -n "id:" <config-path>
+```
+
+Safe fix: pass an existing `--pipeline` value or correct the pipeline id in config.
+
+Reference: [Configuration](config.md#pipelines).
+
+## Source Bundles Are Not Found
+
+Symptom: `no bundles found`, `no bundles found under "."`, `stat ssh ... not_found`, or `stat s3 ... not_found`.
+
+Likely cause: the source root, source-root-relative bundle path, S3 prefix, SSH path, or permissions do not expose a directory containing `manifest.json`.
+
+Diagnostic:
+
+```sh
+go run ./cmd/distributor validate --config <config-path> --pipeline <pipeline-id>
+```
+
+Safe fix: correct the configured source backend root, permissions, prefix, or `--bundle` path. The selected bundle directory must contain `manifest.json`.
+
+Reference: [Operations](operations.md#filesystem-and-storage-layout).
+
+## Source Manifest Or Files Fail Validation
+
+Symptom: `sha256 mismatch`, `size mismatch`, `digest mismatch`, missing manifest fields, or unsafe source paths.
+
+Likely cause: files changed after `manifest.json` was written, the manifest digest is stale, or the producer wrote invalid bundle paths.
 
 Diagnostic:
 
@@ -83,11 +145,15 @@ Diagnostic:
 go run ./cmd/distributor validate <source-root>
 ```
 
-Safe fix: regenerate the producer bundle and manifest together. Do not edit destination state to work around source digest failures.
+Safe fix: regenerate the producer bundle and manifest together. Do not edit destination state to work around source validation failures.
 
-## `destination has content but no distributor state`
+Reference: [Operations](operations.md#cleanup-and-recovery).
 
-Likely cause: the destination path is not empty and has no `.distributor.json` state file, so `distributor` will not claim it as managed.
+## Destination Has Unmanaged Content
+
+Symptom: `destination has content but no distributor state` or a plan reason containing `fail_unmanaged`.
+
+Likely cause: the destination bundle path contains files but no valid `.distributor.json`, so `distributor` will not claim it by default.
 
 Diagnostic:
 
@@ -95,11 +161,15 @@ Diagnostic:
 find <destination-path> -maxdepth 2 -print
 ```
 
-Safe fix: choose an empty destination path or move existing files aside after confirming they are not needed. There is no force overwrite option.
+Safe fix: choose an empty destination path, move unrelated files aside, or preview `run --dry-run --force` only after confirming the reported bundle path is safe to replace.
 
-## `fail_conflict`
+Reference: [Operations](operations.md#forced-replacement-workflow).
 
-Likely cause: existing `.distributor.json` belongs to a different pipeline, a different destination, a different source id, or a same-created source with a different digest.
+## Destination State Conflicts With Source
+
+Symptom: `fail_conflict`, `destination source id differs`, `same id and created time but different digest`, `pipeline id ... does not match`, or `destination id ... does not match`.
+
+Likely cause: `.distributor.json` belongs to a different pipeline, destination, source id, or same-created source with different content.
 
 Diagnostic:
 
@@ -108,11 +178,47 @@ cat <destination-path>/.distributor.json
 go run ./cmd/distributor inspect <source-root>
 ```
 
-Safe fix: verify you are publishing the intended source to the intended destination. Use a separate destination path for unrelated content.
+Safe fix: verify the source and destination are intended to match. Use a separate destination path for unrelated content. To replace the existing state, configure `transfer.on_conflict: replace`, preview with `--dry-run --force`, then publish with `--force`.
 
-## `destination output path collision`
+Reference: [Operations](operations.md#destination-state-and-retry-behavior).
 
-Likely cause: configured publication would write two outputs to the same destination path, such as publishing a source `report.html` while also generating `report.html` from `report.md`.
+## Destination Is Newer Than Source
+
+Symptom: `skip_destination_newer` or `destination is newer and replacement requires --force`.
+
+Likely cause: the destination state records a source manifest with a later `created` timestamp than the current source.
+
+Diagnostic:
+
+```sh
+go run ./cmd/distributor run --config <config-path> --dry-run --format json
+```
+
+Safe fix: keep the default skip behavior unless replacement is intentional. To replace newer state, configure `transfer.on_destination_newer: replace`, preview with `--dry-run --force`, then publish with `--force`.
+
+Reference: [Operations](operations.md#forced-replacement-workflow).
+
+## Forced Replacement Appears In A Plan
+
+Symptom: dry-run output includes `force_replace`.
+
+Likely cause: the run used `--force`, and planning selected a supported destructive replacement.
+
+Diagnostic:
+
+```sh
+go run ./cmd/distributor run --config <config-path> --dry-run --force
+```
+
+Safe fix: inspect the pipeline id, destination id, backend, and bundle path. Proceed only if deleting everything inside that destination bundle path is intended.
+
+Reference: [Operations](operations.md#forced-replacement-workflow).
+
+## Output Path Collision
+
+Symptom: `destination output path collision`.
+
+Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`.
 
 Diagnostic:
 
@@ -120,11 +226,15 @@ Diagnostic:
 go run ./cmd/distributor run --config <config-path> --dry-run
 ```
 
-Safe fix: adjust the source bundle contents or publish policy so source and generated outputs do not collide.
+Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide.
 
-## A run failed after writing some files
+Reference: [Configuration](config.md#publish-and-transform-policy).
 
-Likely cause: a write failed partway through publication. Local execution attempts to clean up outputs written during the failed attempt.
+## Run Failed After Writing Some Files
+
+Symptom: a destination write failed and the command exited non-zero after partial work.
+
+Likely cause: storage write failure, permission issue, network interruption, or object-store error during publish execution.
 
 Diagnostic:
 
@@ -132,4 +242,248 @@ Diagnostic:
 find <destination-path> -maxdepth 2 -print
 ```
 
-Safe fix: inspect the destination before retrying. If only unrelated unmanaged files remain, move them aside or choose a clean destination. Re-run with `--dry-run` before publishing again. See [operations](operations.md).
+Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. Rerun `--dry-run` before publishing again.
+
+Reference: [Operations](operations.md#destination-state-and-retry-behavior).
+
+## JSON Run Result Has `ok: false`
+
+Symptom: `run --format json` exits non-zero with a JSON result where `ok` is `false`.
+
+Likely cause: at least one destination failed after planning or execution began, while other destination results were still available.
+
+Diagnostic:
+
+```sh
+go run ./cmd/distributor run --config <config-path> --format json
+```
+
+Safe fix: inspect the top-level `errors` array, destination actions, output errors, and summary. Fix failed destinations, then preview with `--dry-run --format json`.
+
+Reference: [CLI](cli.md#output-and-exit-behavior).
+
+## Secrets Directory Is Missing Or Unreadable
+
+Symptom: `load secrets directory ... no such file or directory`, `permission denied`, or `secret filename ... is invalid`.
+
+Likely cause: `secrets.directory` points to a missing or unreadable directory, or it contains a regular file whose name is not a valid credential variable name.
+
+Diagnostic:
+
+```sh
+ls -ld <secrets-directory>
+find <secrets-directory> -maxdepth 1 -type f -printf '%f\n'
+```
+
+Safe fix: mount or create the directory, adjust permissions for the service user, or rename/remove invalid secret files. Secret filenames must match `[A-Za-z_][A-Za-z0-9_]*`.
+
+Reference: [Configuration](config.md#secrets).
+
+## Credential Variable Is Missing Or Empty
+
+Symptom: `credential environment variable ... is not set`, `credential environment variable ... is empty`, or S3 authentication errors such as `AccessDenied`, `InvalidAccessKeyId`, or `SignatureDoesNotMatch`.
+
+Likely cause: configured S3 credential variable names are not available through the process environment or `secrets.directory`, are empty, or do not authorize the requested bucket/prefix.
+
+Diagnostic:
+
+```sh
+env | cut -d= -f1 | rg '^(<access-key-variable>|<secret-key-variable>)$'
+ls -l <secrets-directory>
+```
+
+Safe fix: provide both configured S3 credential values, correct IAM/service permissions, or omit explicit credential fields to use the AWS SDK default credential chain.
+
+Reference: [Configuration](config.md#s3-compatible-backend).
+
+## Secret File Is Ignored In Favor Of Environment
+
+Symptom: `secret ... ignored because the real environment already has that variable`.
+
+Likely cause: the same credential name exists in the process environment and `secrets.directory` with different values.
+
+Diagnostic:
+
+```sh
+env | cut -d= -f1 | rg '^<variable-name>$'
+ls -l <secrets-directory>/<variable-name>
+```
+
+Safe fix: remove one source of the credential or intentionally keep the process environment value. `distributor` does not print either value.
+
+Reference: [Operations](operations.md#secrets-operation).
+
+## SSH Auth Is Not Configured
+
+Symptom: `no SSH auth methods configured`.
+
+Likely cause: no SSH agent is available and `ssh_key_file` is missing or unreadable.
+
+Diagnostic:
+
+```sh
+test -n "$SSH_AUTH_SOCK" && ssh-add -l
+ls -l <ssh-key-file>
+```
+
+Safe fix: start an SSH agent with an appropriate key loaded, or configure a readable private key with `ssh_key_file`.
+
+Reference: [Configuration](config.md#sshsftp-backend).
+
+## SSH Host Key Fails
+
+Symptom: `host key ... is unknown`, `known_hosts is required`, or `host key ... has changed`.
+
+Likely cause: strict host key checking has no trusted key, `accept-new` cannot persist a new key, or the remote host key differs from the stored key.
+
+Diagnostic:
+
+```sh
+ls -l <known-hosts-path>
+ssh-keygen -F <host> -f <known-hosts-path>
+```
+
+Safe fix: pre-populate `known_hosts` for `strict`, configure a writable `known_hosts` path for `accept-new`, or verify the server identity before updating a changed key. Do not disable host key checking to bypass an unexpected changed key.
+
+Reference: [Operations](operations.md#sshsftp).
+
+## S3 Prefix Is Invalid
+
+Symptom: `prefix must be a clean relative slash-separated path`.
+
+Likely cause: the S3 prefix contains traversal, dot segments, empty segments, or backslashes after leading and trailing slashes are trimmed.
+
+Diagnostic:
+
+```sh
+go run ./cmd/distributor run --config <config-path> --dry-run
+```
+
+Safe fix: use a clean relative prefix such as `reports/archive`, or omit `prefix`.
+
+Reference: [Configuration](config.md#s3-compatible-backend).
+
+## S3 Location Or Connectivity Fails
+
+Symptom: `NoSuchBucket`, `InvalidBucketName`, `not_found`, endpoint connection failures, or TLS/network errors.
+
+Likely cause: endpoint, bucket, prefix, region, path-style mode, network routing, or credentials are wrong for the service.
+
+Diagnostic:
+
+```sh
+go run ./cmd/distributor run --config <config-path> --dry-run
+curl -I <endpoint>
+```
+
+Safe fix: verify `endpoint`, `bucket`, `region`, `prefix`, and `force_path_style`. For S3-compatible services, keep `force_path_style: true` unless the service requires virtual-host addressing. Distributor does not provide an insecure TLS bypass setting.
+
+Reference: [Operations](operations.md#s3-compatible-storage).
+
+## HTTP Server Cannot Bind
+
+Symptom: `bind HTTP server ... address already in use`.
+
+Likely cause: another process is listening on `server.http.bind`.
+
+Diagnostic:
+
+```sh
+ss -ltnp | rg '<port>'
+```
+
+Safe fix: stop the conflicting process or configure a different bind address.
+
+Reference: [Configuration](config.md#serverhttp).
+
+## HTTP Upload Token Is Missing Or Duplicated
+
+Symptom: `upload token environment variable ... is not set`, `... is empty`, or `upload token environment variables ... resolve to the same value`.
+
+Likely cause: an `http_upload` source references a missing/empty `token_env`, or two upload pipelines resolve to the same bearer token.
+
+Diagnostic:
+
+```sh
+rg -n 'token_env:' <config-path>
+env | cut -d= -f1 | rg '^<token-variable>$'
+ls -l <secrets-directory>/<token-variable>
+```
+
+Safe fix: provide one distinct non-empty token value per upload pipeline through the process environment or `secrets.directory`. Do not put literal tokens in YAML.
+
+Reference: [Configuration](config.md#http-upload-source-backend).
+
+## Upload Request Is Unauthorized
+
+Symptom: `POST /upload` returns `401`.
+
+Likely cause: the request lacks `Authorization: Bearer <token>`, has an empty token, or uses a token that does not match any configured upload pipeline.
+
+Diagnostic:
+
+```sh
+curl -i -X POST http://127.0.0.1:8080/upload \
+  -H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
+  -H "Content-Type: application/x-tar" \
+  --data-binary @bundle.tar
+```
+
+Safe fix: use the token value resolved by the configured `token_env`. Do not include token values in logs or tickets.
+
+Reference: [Operations](operations.md#http-upload-operation).
+
+## Upload Request Is Rejected Before A Run ID
+
+Symptom: `POST /upload` returns `400`, `413`, `415`, or `503`.
+
+Likely cause: the request included a `pipeline` or `pipeline_id` query, archive content is malformed, the body exceeds size limits, content type is unsupported, or the in-memory upload queue is full.
+
+Diagnostic:
+
+```sh
+tar -tf bundle.tar
+tar -tzf bundle.tar.gz
+rg -n 'max_upload_size|queue_size|max_concurrency' <config-path>
+```
+
+Safe fix: send one valid tar or tar.gz source bundle archive with `Content-Type: application/x-tar`, `application/gzip`, or `application/x-gzip`; remove pipeline query parameters; reduce archive size or raise the configured limit; retry after queue pressure drops.
+
+Reference: [Operations](operations.md#http-upload-operation).
+
+## Upload Idempotency Conflict
+
+Symptom: `POST /upload` returns `409`.
+
+Likely cause: the request reused an `Idempotency-Key` for the same authenticated pipeline with a different source manifest, or another request with the same key is still being staged before its manifest is known.
+
+Diagnostic:
+
+```sh
+curl -i -X POST http://127.0.0.1:8080/upload \
+  -H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
+  -H "Content-Type: application/gzip" \
+  -H "Idempotency-Key: <key>" \
+  --data-binary @bundle.tar.gz
+```
+
+Safe fix: if the response includes `"retryable":true`, retry the same upload later with the same key. Otherwise, inspect the producer operation and use the same key only for the same source bundle.
+
+Reference: [HTTP Upload API Contract](integrations/http-upload.md#post-upload).
+
+## Upload Status Is Missing
+
+Symptom: `GET /runs/<run_id>` returns `404`.
+
+Likely cause: the run id is wrong, the process restarted, or the retained status record expired after `server.http.retention`.
+
+Diagnostic:
+
+```sh
+curl -i http://127.0.0.1:8080/runs/<run-id>
+rg -n 'retention:' <config-path>
+```
+
+Safe fix: use the exact `run_id` returned by upload admission. Increase retention if operators need a longer status window.
+
+Reference: [Operations](operations.md#http-upload-operation).

examples/archive-and-latest.yml

@@ -0,0 +1,29 @@
+pipelines:
+  - id: example-archive-and-latest
+    source:
+      backend: local
+      path: examples/source-bundle
+    destinations:
+      - id: local-source-archive
+        backend: local
+        path: workspace/published/archive-and-latest/archive
+        path_mapping:
+          mode: preserve_relative
+        publish:
+          source: true
+          html: false
+      - id: local-html-latest
+        backend: local
+        path: workspace/published/archive-and-latest/latest
+        path_mapping:
+          mode: fixed
+        links:
+          base_url: https://reports.example.com/latest
+          primary: auto
+        publish:
+          source: false
+          html: true
+        transform:
+          markdown_to_html:
+            enabled: true
+            mode: index

examples/http-upload-local.yml

@@ -0,0 +1,24 @@
+# Local HTTP upload example.
+# Set DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN in the process environment or provide a
+# secrets-directory file with that name before running `distributor serve`.
+server:
+  http:
+    bind: 127.0.0.1:8080
+    staging_root: workspace/http-upload/staging
+    max_upload_size: 20MB
+    queue_size: 16
+    max_concurrency: 1
+    retention: 24h
+pipelines:
+  - id: example-http-upload
+    source:
+      backend: http_upload
+      token_env: DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN
+    destinations:
+      - id: local-archive
+        backend: local
+        path: workspace/published/http-upload
+        publish:
+          source: true
+          html: false
+

examples/local-index.yml

@@ -0,0 +1,16 @@
+pipelines:
+  - id: example-index-bundle
+    source:
+      backend: local
+      path: examples/source-bundle
+    destinations:
+      - id: local-index
+        backend: local
+        path: workspace/published/index-bundle
+        publish:
+          source: false
+          html: true
+        transform:
+          markdown_to_html:
+            enabled: true
+            mode: index

examples/s3-destination.yml

@@ -0,0 +1,22 @@
+# Environment-gated example.
+# Replace endpoint, bucket, prefix, and credential environment variable names
+# with values for an S3-compatible service you control before running this config.
+pipelines:
+  - id: example-s3-destination
+    source:
+      backend: local
+      path: examples/source-bundle
+    destinations:
+      - id: s3-archive
+        backend: s3
+        endpoint: https://s3.example.com
+        bucket: reports
+        prefix: distributor/archive
+        region: us-east-1
+        force_path_style: true
+        credentials:
+          access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
+          secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
+        publish:
+          source: true
+          html: false

examples/ssh-destination.yml

@@ -0,0 +1,21 @@
+# Environment-gated example.
+# Replace host, user, path, ssh_key_file, and known_hosts with values for an
+# SSH/SFTP endpoint you control before running this config.
+pipelines:
+  - id: example-ssh-destination
+    source:
+      backend: local
+      path: examples/source-bundle
+    destinations:
+      - id: ssh-archive
+        backend: ssh
+        host: ssh.example.com
+        user: distributor
+        port: 22
+        path: /srv/distributor/archive
+        ssh_key_file: /home/distributor/.ssh/id_ed25519
+        known_hosts: /home/distributor/.ssh/known_hosts
+        host_key_policy: strict
+        publish:
+          source: true
+          html: false

examples/upload-client/main.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/pkg/upload"
+)
+
+func main() {
+	token := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN")
+	if token == "" {
+		log.Fatal("set DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN before running this example")
+	}
+	endpoint := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "http://127.0.0.1:8080"
+	}
+	bundleRoot := "examples/source-bundle"
+	if len(os.Args) > 1 {
+		bundleRoot = os.Args[1]
+	}
+	idempotencyKey := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY")
+
+	client, err := upload.NewClient(upload.ClientOptions{
+		Endpoint: endpoint,
+		Token:    token,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	opts := upload.UploadBundleOptions{Root: bundleRoot}
+	if idempotencyKey != "" {
+		opts.IdempotencyKey = idempotencyKey
+	}
+	result, err := client.UploadBundle(ctx, opts)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("accepted run %s with status %s\n", result.RunID, result.Status)
+}

go.mod

@@ -3,6 +3,31 @@ module gitea.maximumdirect.net/eric/distributor
 go 1.26
 
 require (
+	github.com/aws/aws-sdk-go-v2 v1.41.9
+	github.com/aws/aws-sdk-go-v2/config v1.32.20
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.19
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2
+	github.com/aws/smithy-go v1.26.0
+	github.com/pkg/sftp v1.13.10
 	github.com/yuin/goldmark v1.8.2
+	golang.org/x/crypto v0.52.0
 	gopkg.in/yaml.v3 v3.0.1
 )
+
+require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	golang.org/x/sys v0.45.0 // indirect
+)

go.sum

@@ -1,5 +1,57 @@
+github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4=
+github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94=
+github.com/aws/aws-sdk-go-v2/config v1.32.20 h1:8VMDnWc/kEzxsI/1ngGM9mG81a8IGmIHD8KLcYGwagc=
+github.com/aws/aws-sdk-go-v2/config v1.32.20/go.mod h1:PuwEpciweIXGULWeOeSTXtSbH4CW9mWdWrhdCKQI1sM=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.19 h1:yuFzSV1U0aRNYCQGVaTY2zW2M/L93pYHnXnrJUphYhU=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.19/go.mod h1:7y63L1kGzeoDlJaQ3Z578KrnmfBut96JjvJUzGwR+YE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 h1:0w6dCiO8iez+YKwRhRBlL1CH/E3GTfdkuzrwj1by8vo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25/go.mod h1:9FDWUothyr5RCRAHc45XOiVCzUR8n/IhCYX+uVqw6vk=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 h1:Uii3frf9ztec/ABM2/FSH9/z7PLzxfpG8h4RpkUFflQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25/go.mod h1:G6kntsA2GorAxDPbap6xgB2F+amSLUF8GJTi7PUoX44=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 h1:r1+/l6m+WaUJF9HISEsNOLHSNj5EXYQxK8VX6Cz9NlA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25/go.mod h1:cKf+D+NMDK1LndD7BowHbBZPgR9V0/5HubH0PFWvA+c=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 h1:A1PmWU2zfkIm9EyFlJncFXL4W4phML+h8KjltUsCvNQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26/go.mod h1:dY4MRzXEizrD4hqtpKvWVGPX7QleSGGVY+EBolo1RmM=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 h1:d5/908OJ4bXg8lyjeMPvXetEKqoDoLi5Owy1zNue3yg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 h1:dD3dhHNglpd98gs72my22Ndqi1hqQGllFFg1F+twfxg=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25/go.mod h1:0yAbjPfd64gG7mj85RW+fMEYdfBgCRZw8g/oWcL1pjc=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0=
+github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 h1:1VwbP3qMNfxUDEXWki4rCE5iA+44VA1lokTz9HasGzw=
+github.com/aws/aws-sdk-go-v2/service/signin v1.1.1/go.mod h1:vUtyoSj0OPji3kjIVSc/GlKuWEiL33f/WFxl6dmpy/A=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 h1:N6pIsdFOW1Kd9S4KyFKXdGRBojPPxkP32+uHFWLv4Hc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.19/go.mod h1:3gt5WJArFooNmyLONS+h/R4J+o86II8du38IgCwj9dE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 h1:hc+lBYiiTr8Zk4MTzIsQ92MeDWCIDvWGmzKUWOaBcOg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2/go.mod h1:hU6fqB3OJA6/ePheD47LQnxvjYk6br6PtQxs+Q9ojvk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 h1:ErklX/7uhSbkAAeyQD/Y1OoQ9hO3SJXQNEgksORW3Js=
+github.com/aws/aws-sdk-go-v2/service/sts v1.42.3/go.mod h1:ULe4HCzfKPiR6R3HEurE3b1upEkuk8AkMrOKtaOxKO8=
+github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s=
+github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
+github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
 github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/adapters/local/backend.go

@@ -161,31 +161,10 @@ func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOpti
 		}
 		return b.translateError(storage.OpWalk, prefix, err)
 	}
-	visited := 0
-	emit := func(entry storage.Entry) error {
-		if err := ctx.Err(); err != nil {
-			return err
-		}
-		if opts.Limit > 0 && visited >= opts.Limit {
-			return storage.ErrStopWalk
-		}
-		visited++
-		if err := fn(entry); err != nil {
-			if errors.Is(err, storage.ErrStopWalk) {
-				return storage.ErrStopWalk
-			}
-			return storage.NewError(storage.OpWalk, backendName, entry.Path, storage.ErrUnknown, err)
-		}
-		return nil
-	}
+	emitter := storage.NewWalkEmitter(ctx, backendName, opts, fn)
 
 	if !info.IsDir() {
-		if err := emit(entryFromInfo(prefix, info)); errors.Is(err, storage.ErrStopWalk) {
-			return nil
-		} else if err != nil {
-			return err
-		}
-		return nil
+		return storage.FinishWalk(emitter.Emit(entryFromInfo(prefix, info)))
 	}
 
 	walkErr := filepath.WalkDir(nativePrefix, func(nativePath string, dirEntry fs.DirEntry, err error) error {
@@ -210,24 +189,13 @@ func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOpti
 		if err != nil {
 			return b.translateError(storage.OpWalk, logicalPath, err)
 		}
-		return emit(entryFromInfo(logicalPath, info))
+		return emitter.Emit(entryFromInfo(logicalPath, info))
 	})
-	if errors.Is(walkErr, storage.ErrStopWalk) {
-		return nil
-	}
-	return walkErr
+	return storage.FinishWalk(walkErr)
 }
 
 func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
-	found := false
-	err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
-		found = true
-		return storage.ErrStopWalk
-	})
-	if err != nil {
-		return false, err
-	}
-	return found, nil
+	return storage.HasAny(ctx, b, prefix)
 }
 
 func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
@@ -267,6 +235,51 @@ func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, ma
 	return nil
 }
 
+func (b *Backend) DeletePrefix(ctx context.Context, prefix string, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	nativePrefix, err := b.nativePath(prefix, true)
+	if err != nil {
+		return err
+	}
+	if err := b.rejectSymlinkAncestors(nativePrefix, false); err != nil {
+		return err
+	}
+	if prefix == "" {
+		entries, err := os.ReadDir(nativePrefix)
+		if err != nil {
+			if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
+				return nil
+			}
+			return b.translateError(storage.OpDeletePrefix, prefix, err)
+		}
+		for _, entry := range entries {
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+			child := filepath.Join(nativePrefix, entry.Name())
+			if err := os.RemoveAll(child); err != nil {
+				return b.translateError(storage.OpDeletePrefix, entry.Name(), err)
+			}
+		}
+		return nil
+	}
+	if _, err := os.Lstat(nativePrefix); err != nil {
+		if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
+			return nil
+		}
+		return b.translateError(storage.OpDeletePrefix, prefix, err)
+	}
+	if err := os.RemoveAll(nativePrefix); err != nil {
+		return b.translateError(storage.OpDeletePrefix, prefix, err)
+	}
+	if opts.PruneEmptyDirs {
+		b.pruneEmptyParents(filepath.Dir(nativePrefix))
+	}
+	return nil
+}
+
 func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) {
 	if logicalPath == "" {
 		if !allowEmpty {

internal/adapters/local/backend_test.go

@@ -169,6 +169,30 @@ func TestBackendManagedDeletion(t *testing.T) {
 	}
 }
 
+func TestBackendDeletePrefixStaysWithinPrefix(t *testing.T) {
+	backend := newBackend(t)
+	mustWrite(t, backend, "bundle/report.md", "report")
+	mustWrite(t, backend, "bundle/nested/old.txt", "old")
+	mustWrite(t, backend, "bundle-sibling/keep.txt", "keep")
+	mustWrite(t, backend, "outside.txt", "outside")
+
+	if err := backend.DeletePrefix(context.Background(), "bundle", storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
+		t.Fatalf("DeletePrefix() error = %v", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle/report.md"); !storage.IsNotFound(err) {
+		t.Fatalf("deleted file stat error = %v, want not found", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle/nested/old.txt"); !storage.IsNotFound(err) {
+		t.Fatalf("deleted nested file stat error = %v, want not found", err)
+	}
+	if _, err := backend.Stat(context.Background(), "bundle-sibling/keep.txt"); err != nil {
+		t.Fatalf("sibling stat error = %v", err)
+	}
+	if _, err := backend.Stat(context.Background(), "outside.txt"); err != nil {
+		t.Fatalf("outside stat error = %v", err)
+	}
+}
+
 func TestBackendHasAny(t *testing.T) {
 	backend := newBackend(t)
 	found, err := backend.HasAny(context.Background(), "missing")

internal/adapters/s3/backend.go

@@ -0,0 +1,455 @@
+package s3
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"sort"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awscfg "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+)
+
+type Client interface {
+	HeadObject(ctx context.Context, input *awss3.HeadObjectInput, optFns ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error)
+	GetObject(ctx context.Context, input *awss3.GetObjectInput, optFns ...func(*awss3.Options)) (*awss3.GetObjectOutput, error)
+	PutObject(ctx context.Context, input *awss3.PutObjectInput, optFns ...func(*awss3.Options)) (*awss3.PutObjectOutput, error)
+	ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, optFns ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error)
+	DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, optFns ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error)
+}
+
+type Backend struct {
+	client Client
+	bucket string
+	prefix string
+}
+
+func New(ctx context.Context, options Options) (*Backend, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	options, err := options.normalized()
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Prefix, storage.ErrInvalidPath, err)
+	}
+	loadOptions := []func(*awscfg.LoadOptions) error{
+		awscfg.WithRegion(options.Region),
+	}
+	if options.AccessKeyID != "" {
+		loadOptions = append(loadOptions, awscfg.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(options.AccessKeyID, options.SecretAccessKey, "")))
+	}
+	cfg, err := awscfg.LoadDefaultConfig(ctx, loadOptions...)
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Endpoint, storage.ErrUnknown, err)
+	}
+	client := awss3.NewFromConfig(cfg, func(o *awss3.Options) {
+		o.BaseEndpoint = aws.String(options.Endpoint)
+		o.UsePathStyle = options.ForcePathStyle
+	})
+	return NewWithClient(client, options)
+}
+
+func NewWithClient(client Client, options Options) (*Backend, error) {
+	options, err := options.normalized()
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Prefix, storage.ErrInvalidPath, err)
+	}
+	if client == nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Bucket, storage.ErrInvalidPath, fmt.Errorf("client is required"))
+	}
+	return &Backend{
+		client: client,
+		bucket: options.Bucket,
+		prefix: options.Prefix,
+	}, nil
+}
+
+func (b *Backend) ReadFile(ctx context.Context, logicalPath string) ([]byte, error) {
+	reader, err := b.OpenReader(ctx, logicalPath)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, storage.NewError(storage.OpReadFile, BackendName, logicalPath, storage.ErrUnknown, err)
+	}
+	return data, nil
+}
+
+func (b *Backend) OpenReader(ctx context.Context, logicalPath string) (io.ReadCloser, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	key, err := b.objectKey(logicalPath, false)
+	if err != nil {
+		return nil, err
+	}
+	output, err := b.client.GetObject(ctx, &awss3.GetObjectInput{
+		Bucket: aws.String(b.bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
+	}
+	return output.Body, nil
+}
+
+func (b *Backend) WriteFile(ctx context.Context, logicalPath string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
+	opts.Size = int64(len(data))
+	opts.SizeKnown = true
+	return b.WriteFrom(ctx, logicalPath, bytes.NewReader(data), opts)
+}
+
+func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	key, err := b.objectKey(logicalPath, false)
+	if err != nil {
+		return storage.Entry{}, err
+	}
+	if !opts.Overwrite {
+		_, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
+			Bucket: aws.String(b.bucket),
+			Key:    aws.String(key),
+		})
+		if err == nil {
+			return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrAlreadyExist, nil)
+		}
+		if !isNotFound(err) {
+			return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
+		}
+	}
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, err)
+	}
+	if opts.SizeKnown && int64(len(data)) != opts.Size {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", len(data), opts.Size))
+	}
+	contentType := opts.ContentType
+	if contentType == "" {
+		contentType = ContentType(logicalPath)
+	}
+	_, err = b.client.PutObject(ctx, &awss3.PutObjectInput{
+		Bucket:        aws.String(b.bucket),
+		Key:           aws.String(key),
+		Body:          bytes.NewReader(data),
+		ContentLength: aws.Int64(int64(len(data))),
+		ContentType:   aws.String(contentType),
+	})
+	if err != nil {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
+	}
+	return storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: int64(len(data))}, nil
+}
+
+func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	key, err := b.objectKey(logicalPath, false)
+	if err != nil {
+		return storage.Entry{}, err
+	}
+	output, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
+		Bucket: aws.String(b.bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		return storage.Entry{}, b.translateError(storage.OpStat, logicalPath, err)
+	}
+	size := int64(0)
+	if output.ContentLength != nil {
+		size = *output.ContentLength
+	}
+	return storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: size}, nil
+}
+
+func (b *Backend) Walk(ctx context.Context, logicalPrefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if err := storage.ValidatePrefix(logicalPrefix); err != nil {
+		return err
+	}
+	emitter := storage.NewWalkEmitter(ctx, BackendName, opts, fn)
+	if logicalPrefix != "" {
+		entry, err := b.Stat(ctx, logicalPrefix)
+		if err == nil {
+			if err := emitter.Emit(entry); err != nil {
+				return storage.FinishWalk(err)
+			}
+			if emitter.LimitReached() {
+				return nil
+			}
+		} else if !storage.IsNotFound(err) {
+			return err
+		}
+	}
+	return storage.FinishWalk(b.walkObjects(ctx, logicalPrefix, opts, emitter.Emit))
+}
+
+func (b *Backend) HasAny(ctx context.Context, logicalPrefix string) (bool, error) {
+	return storage.HasAny(ctx, b, logicalPrefix)
+}
+
+func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
+	if err != nil {
+		return err
+	}
+	for _, target := range targets {
+		if err := b.deleteObject(ctx, storage.OpDeleteManagedBundle, target, opts); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (b *Backend) DeletePrefix(ctx context.Context, logicalPrefix string, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if err := storage.ValidatePrefix(logicalPrefix); err != nil {
+		return err
+	}
+	found := false
+	if logicalPrefix != "" {
+		key, err := b.objectKey(logicalPrefix, false)
+		if err != nil {
+			return err
+		}
+		_, err = b.client.HeadObject(ctx, &awss3.HeadObjectInput{
+			Bucket: aws.String(b.bucket),
+			Key:    aws.String(key),
+		})
+		if err == nil {
+			found = true
+			if err := b.deleteObject(ctx, storage.OpDeletePrefix, logicalPrefix, storage.DeleteOptions{IgnoreMissing: false}); err != nil {
+				return err
+			}
+		} else if !isNotFound(err) {
+			return b.translateError(storage.OpDeletePrefix, logicalPrefix, err)
+		}
+	}
+	var entries []storage.Entry
+	if err := b.walkObjects(ctx, logicalPrefix, storage.WalkOptions{Recursive: true}, func(entry storage.Entry) error {
+		if entry.Type == storage.EntryTypeFile {
+			entries = append(entries, entry)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		found = true
+		if err := b.deleteObject(ctx, storage.OpDeletePrefix, entry.Path, storage.DeleteOptions{IgnoreMissing: true}); err != nil {
+			return err
+		}
+	}
+	if !found && !opts.IgnoreMissing {
+		return storage.NewError(storage.OpDeletePrefix, BackendName, logicalPrefix, storage.ErrNotFound, nil)
+	}
+	return nil
+}
+
+func (b *Backend) deleteObject(ctx context.Context, op, logicalPath string, opts storage.DeleteOptions) error {
+	key, err := b.objectKey(logicalPath, false)
+	if err != nil {
+		return err
+	}
+	if key == b.prefix {
+		return storage.NewError(op, BackendName, logicalPath, storage.ErrInvalidPath, nil)
+	}
+	if !opts.IgnoreMissing {
+		_, err := b.client.HeadObject(ctx, &awss3.HeadObjectInput{
+			Bucket: aws.String(b.bucket),
+			Key:    aws.String(key),
+		})
+		if err != nil {
+			return b.translateError(op, logicalPath, err)
+		}
+	}
+	_, err = b.client.DeleteObject(ctx, &awss3.DeleteObjectInput{
+		Bucket: aws.String(b.bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		if opts.IgnoreMissing && isNotFound(err) {
+			return nil
+		}
+		return b.translateError(op, logicalPath, err)
+	}
+	return nil
+}
+
+func (b *Backend) walkObjects(ctx context.Context, logicalPrefix string, opts storage.WalkOptions, emit func(storage.Entry) error) error {
+	listPrefix, err := b.listPrefix(logicalPrefix)
+	if err != nil {
+		return err
+	}
+	delimiter := ""
+	if !opts.Recursive {
+		delimiter = "/"
+	}
+	var token *string
+	for {
+		output, err := b.client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{
+			Bucket:            aws.String(b.bucket),
+			Prefix:            aws.String(listPrefix),
+			Delimiter:         aws.String(delimiter),
+			ContinuationToken: token,
+		})
+		if err != nil {
+			return b.translateError(storage.OpWalk, logicalPrefix, err)
+		}
+		entries := entriesFromList(logicalPrefix, b.prefix, output)
+		sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path })
+		for _, entry := range entries {
+			if entry.Path == "" {
+				continue
+			}
+			if err := emit(entry); err != nil {
+				return err
+			}
+		}
+		if output.IsTruncated == nil || !*output.IsTruncated {
+			return nil
+		}
+		token = output.NextContinuationToken
+	}
+}
+
+func entriesFromList(logicalPrefix, rootPrefix string, output *awss3.ListObjectsV2Output) []storage.Entry {
+	seen := make(map[string]storage.Entry)
+	for _, object := range output.Contents {
+		if object.Key == nil {
+			continue
+		}
+		logicalPath := logicalPathFromKey(rootPrefix, *object.Key)
+		if logicalPath == "" || logicalPath == logicalPrefix {
+			continue
+		}
+		size := int64(0)
+		if object.Size != nil {
+			size = *object.Size
+		}
+		seen[logicalPath] = storage.Entry{Path: logicalPath, Type: storage.EntryTypeFile, Size: size}
+	}
+	for _, commonPrefix := range output.CommonPrefixes {
+		if commonPrefix.Prefix == nil {
+			continue
+		}
+		logicalPath := strings.TrimSuffix(logicalPathFromKey(rootPrefix, *commonPrefix.Prefix), "/")
+		if logicalPath == "" || logicalPath == logicalPrefix {
+			continue
+		}
+		seen[logicalPath] = storage.Entry{Path: logicalPath, Type: storage.EntryTypeDirectory}
+	}
+	entries := make([]storage.Entry, 0, len(seen))
+	for _, entry := range seen {
+		entries = append(entries, entry)
+	}
+	return entries
+}
+
+func (b *Backend) objectKey(logicalPath string, allowEmpty bool) (string, error) {
+	if logicalPath == "" {
+		if !allowEmpty {
+			return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
+		}
+		return b.prefix, nil
+	}
+	if err := storage.ValidatePath(logicalPath); err != nil {
+		return "", err
+	}
+	if b.prefix == "" {
+		return logicalPath, nil
+	}
+	return b.prefix + "/" + logicalPath, nil
+}
+
+func (b *Backend) listPrefix(logicalPrefix string) (string, error) {
+	key, err := b.objectKey(logicalPrefix, true)
+	if err != nil {
+		return "", err
+	}
+	if key != "" {
+		key = strings.TrimSuffix(key, "/") + "/"
+	}
+	return key, nil
+}
+
+func logicalPathFromKey(rootPrefix, key string) string {
+	if rootPrefix == "" {
+		return key
+	}
+	if key == rootPrefix {
+		return ""
+	}
+	return strings.TrimPrefix(key, rootPrefix+"/")
+}
+
+func ContentType(logicalPath string) string {
+	switch strings.ToLower(path.Ext(logicalPath)) {
+	case ".md":
+		return "text/markdown; charset=utf-8"
+	case ".html":
+		return "text/html; charset=utf-8"
+	case ".json":
+		return "application/json"
+	case ".txt":
+		return "text/plain; charset=utf-8"
+	default:
+		return "application/octet-stream"
+	}
+}
+
+func isNotFound(err error) bool {
+	var notFound *types.NotFound
+	if errors.As(err, &notFound) {
+		return true
+	}
+	var apiErr smithy.APIError
+	if errors.As(err, &apiErr) {
+		switch apiErr.ErrorCode() {
+		case "NotFound", "NoSuchKey", "404":
+			return true
+		}
+	}
+	return false
+}
+
+func (b *Backend) translateError(op, logicalPath string, err error) error {
+	kind := storage.ErrUnknown
+	if isNotFound(err) {
+		kind = storage.ErrNotFound
+	} else {
+		var apiErr smithy.APIError
+		if errors.As(err, &apiErr) {
+			switch apiErr.ErrorCode() {
+			case "AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch":
+				kind = storage.ErrPermission
+			case "SlowDown", "RequestTimeout", "ServiceUnavailable", "InternalError":
+				kind = storage.ErrTemporary
+			case "InvalidBucketName", "NoSuchBucket":
+				kind = storage.ErrInvalidPath
+			}
+		}
+	}
+	return storage.NewError(op, BackendName, logicalPath, kind, err)
+}

internal/adapters/s3/backend_test.go

@@ -0,0 +1,442 @@
+package s3
+
+import (
+	"context"
+	"io"
+	"sort"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+func TestKeyAndPrefixNormalization(t *testing.T) {
+	backend := newTestBackend(t, "root/prefix", nil)
+	key, err := backend.objectKey("bundle/report.md", false)
+	if err != nil {
+		t.Fatalf("objectKey() error = %v", err)
+	}
+	if got, want := key, "root/prefix/bundle/report.md"; got != want {
+		t.Fatalf("objectKey() = %q, want %q", got, want)
+	}
+	listPrefix, err := backend.listPrefix("bundle")
+	if err != nil {
+		t.Fatalf("listPrefix() error = %v", err)
+	}
+	if got, want := listPrefix, "root/prefix/bundle/"; got != want {
+		t.Fatalf("listPrefix() = %q, want %q", got, want)
+	}
+}
+
+func TestPathTraversalRejected(t *testing.T) {
+	backend := newTestBackend(t, "", nil)
+	for _, logicalPath := range []string{"/absolute", "../escape", "a/../b", `a\b`} {
+		t.Run(logicalPath, func(t *testing.T) {
+			if _, err := backend.objectKey(logicalPath, false); err == nil || !storage.IsInvalidPath(err) {
+				t.Fatalf("objectKey() error = %v, want invalid path", err)
+			}
+		})
+	}
+}
+
+func TestContentType(t *testing.T) {
+	tests := map[string]string{
+		"report.md":   "text/markdown; charset=utf-8",
+		"report.html": "text/html; charset=utf-8",
+		"state.json":  "application/json",
+		"summary.txt": "text/plain; charset=utf-8",
+		"data.bin":    "application/octet-stream",
+	}
+	for path, want := range tests {
+		if got := ContentType(path); got != want {
+			t.Fatalf("ContentType(%q) = %q, want %q", path, got, want)
+		}
+	}
+}
+
+func TestStatRequiresExactObject(t *testing.T) {
+	client := newFakeClient(map[string]string{"root/dir/file.txt": "data"})
+	backend := newTestBackend(t, "root", client)
+	_, err := backend.Stat(context.Background(), "dir")
+	if err == nil || !storage.IsNotFound(err) {
+		t.Fatalf("Stat() error = %v, want not found", err)
+	}
+}
+
+func TestWalkIncludesExactObjectAndDescendants(t *testing.T) {
+	client := newFakeClient(map[string]string{
+		"root/bundle":           "marker",
+		"root/bundle/report.md": "report",
+	})
+	backend := newTestBackend(t, "root", client)
+	entries, err := storage.List(context.Background(), backend, "bundle", storage.WalkOptions{Recursive: true})
+	if err != nil {
+		t.Fatalf("List() error = %v", err)
+	}
+	paths := entryPaths(entries)
+	if got, want := paths, []string{"bundle", "bundle/report.md"}; !equalStrings(got, want) {
+		t.Fatalf("paths = %v, want %v", got, want)
+	}
+}
+
+func TestWalkExactObjectHonorsLimitBeforeListingDescendants(t *testing.T) {
+	client := newFakeClient(map[string]string{
+		"root/bundle":           "marker",
+		"root/bundle/report.md": "report",
+	})
+	backend := newTestBackend(t, "root", client)
+	var entries []storage.Entry
+	err := backend.Walk(context.Background(), "bundle", storage.WalkOptions{Recursive: true, Limit: 1}, func(entry storage.Entry) error {
+		entries = append(entries, entry)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Walk() error = %v", err)
+	}
+	paths := entryPaths(entries)
+	if got, want := paths, []string{"bundle"}; !equalStrings(got, want) {
+		t.Fatalf("paths = %v, want %v", got, want)
+	}
+	if len(client.tokens) != 0 {
+		t.Fatalf("list calls = %d, want none", len(client.tokens))
+	}
+}
+
+func TestHasAnyWithExactObjectStopsBeforeListingDescendants(t *testing.T) {
+	client := newFakeClient(map[string]string{
+		"root/bundle":           "marker",
+		"root/bundle/report.md": "report",
+	})
+	backend := newTestBackend(t, "root", client)
+	found, err := backend.HasAny(context.Background(), "bundle")
+	if err != nil {
+		t.Fatalf("HasAny() error = %v", err)
+	}
+	if !found {
+		t.Fatal("HasAny() = false, want true")
+	}
+	if len(client.tokens) != 0 {
+		t.Fatalf("list calls = %d, want none", len(client.tokens))
+	}
+}
+
+func TestWriteFromChecksOverwriteBeforePut(t *testing.T) {
+	client := newFakeClient(map[string]string{"root/report.md": "old"})
+	backend := newTestBackend(t, "root", client)
+	_, err := backend.WriteFile(context.Background(), "report.md", []byte("new"), storage.WriteOptions{})
+	if err == nil || !storage.IsAlreadyExists(err) {
+		t.Fatalf("WriteFile() error = %v, want already exists", err)
+	}
+	if len(client.putKeys) != 0 {
+		t.Fatalf("put keys = %v, want none", client.putKeys)
+	}
+}
+
+func TestWriteFromPutsNewObjectWithContentType(t *testing.T) {
+	client := newFakeClient(nil)
+	backend := newTestBackend(t, "root", client)
+	entry, err := backend.WriteFile(context.Background(), "report.html", []byte("<p>ok</p>"), storage.WriteOptions{})
+	if err != nil {
+		t.Fatalf("WriteFile() error = %v", err)
+	}
+	if entry.Path != "report.html" || entry.Size != 9 || entry.Type != storage.EntryTypeFile {
+		t.Fatalf("entry = %#v", entry)
+	}
+	if got, want := client.objects["root/report.html"], "<p>ok</p>"; got != want {
+		t.Fatalf("object = %q, want %q", got, want)
+	}
+	if got, want := client.contentTypes["root/report.html"], "text/html; charset=utf-8"; got != want {
+		t.Fatalf("content type = %q, want %q", got, want)
+	}
+}
+
+func TestWalkUsesPagination(t *testing.T) {
+	client := newFakeClient(nil)
+	client.listPages = []awss3.ListObjectsV2Output{
+		{
+			Contents:              []types.Object{{Key: aws.String("root/a.txt"), Size: aws.Int64(1)}},
+			IsTruncated:           aws.Bool(true),
+			NextContinuationToken: aws.String("next"),
+		},
+		{
+			Contents:    []types.Object{{Key: aws.String("root/b.txt"), Size: aws.Int64(2)}},
+			IsTruncated: aws.Bool(false),
+		},
+	}
+	backend := newTestBackend(t, "root", client)
+	entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: true})
+	if err != nil {
+		t.Fatalf("List() error = %v", err)
+	}
+	paths := entryPaths(entries)
+	if got, want := paths, []string{"a.txt", "b.txt"}; !equalStrings(got, want) {
+		t.Fatalf("paths = %v, want %v", got, want)
+	}
+	if got, want := client.tokens, []string{"", "next"}; !equalStrings(got, want) {
+		t.Fatalf("tokens = %v, want %v", got, want)
+	}
+}
+
+func TestHasAnyStopsAfterFirstPage(t *testing.T) {
+	client := newFakeClient(nil)
+	client.listPages = []awss3.ListObjectsV2Output{
+		{
+			Contents:              []types.Object{{Key: aws.String("root/a.txt"), Size: aws.Int64(1)}},
+			IsTruncated:           aws.Bool(true),
+			NextContinuationToken: aws.String("next"),
+		},
+		{
+			Contents:    []types.Object{{Key: aws.String("root/b.txt"), Size: aws.Int64(2)}},
+			IsTruncated: aws.Bool(false),
+		},
+	}
+	backend := newTestBackend(t, "root", client)
+	found, err := backend.HasAny(context.Background(), "")
+	if err != nil {
+		t.Fatalf("HasAny() error = %v", err)
+	}
+	if !found {
+		t.Fatal("HasAny() = false, want true")
+	}
+	if got, want := len(client.tokens), 1; got != want {
+		t.Fatalf("list calls = %d, want %d", got, want)
+	}
+}
+
+func TestWalkNonRecursiveUsesPrefixBoundary(t *testing.T) {
+	client := newFakeClient(map[string]string{
+		"base/dir/file.txt": "nested",
+		"base/file.txt":     "file",
+		"baseball/file.txt": "wrong",
+	})
+	backend := newTestBackend(t, "base", client)
+	entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{})
+	if err != nil {
+		t.Fatalf("List() error = %v", err)
+	}
+	paths := entryPaths(entries)
+	if got, want := paths, []string{"dir", "file.txt"}; !equalStrings(got, want) {
+		t.Fatalf("paths = %v, want %v", got, want)
+	}
+}
+
+func TestDeleteManagedBundleDeletesOnlyManagedTargets(t *testing.T) {
+	client := newFakeClient(map[string]string{
+		"root/report.md":         "report",
+		"root/.distributor.json": "state",
+		"root/keep.txt":          "keep",
+	})
+	backend := newTestBackend(t, "root", client)
+	err := backend.DeleteManagedBundle(context.Background(), "", []string{"report.md"}, storage.DeleteOptions{IgnoreMissing: true})
+	if err != nil {
+		t.Fatalf("DeleteManagedBundle() error = %v", err)
+	}
+	if _, ok := client.objects["root/report.md"]; ok {
+		t.Fatal("managed output still exists")
+	}
+	if _, ok := client.objects["root/.distributor.json"]; ok {
+		t.Fatal("state file still exists")
+	}
+	if _, ok := client.objects["root/keep.txt"]; !ok {
+		t.Fatal("unmanaged object was deleted")
+	}
+	if got, want := sortedStrings(client.deleteKeys), []string{"root/.distributor.json", "root/report.md"}; !equalStrings(got, want) {
+		t.Fatalf("deleted keys = %v, want %v", got, want)
+	}
+}
+
+func TestDeletePrefixStaysWithinPrefix(t *testing.T) {
+	client := newFakeClient(map[string]string{
+		"root/bundle/report.md":           "report",
+		"root/bundle/nested/old.txt":      "old",
+		"root/bundle-sibling/keep.txt":    "keep",
+		"root/outside.txt":                "outside",
+		"other-root/bundle/report.md":     "other",
+		"root/.distributor-prefix-marker": "marker",
+	})
+	backend := newTestBackend(t, "root", client)
+	if err := backend.DeletePrefix(context.Background(), "bundle", storage.DeleteOptions{IgnoreMissing: true}); err != nil {
+		t.Fatalf("DeletePrefix() error = %v", err)
+	}
+	for _, deleted := range []string{"root/bundle/report.md", "root/bundle/nested/old.txt"} {
+		if _, ok := client.objects[deleted]; ok {
+			t.Fatalf("%s still exists", deleted)
+		}
+	}
+	for _, kept := range []string{"root/bundle-sibling/keep.txt", "root/outside.txt", "other-root/bundle/report.md", "root/.distributor-prefix-marker"} {
+		if _, ok := client.objects[kept]; !ok {
+			t.Fatalf("%s was deleted", kept)
+		}
+	}
+	if got, want := sortedStrings(client.deleteKeys), []string{"root/bundle/nested/old.txt", "root/bundle/report.md"}; !equalStrings(got, want) {
+		t.Fatalf("deleted keys = %v, want %v", got, want)
+	}
+}
+
+func newTestBackend(t *testing.T, prefix string, client *fakeClient) *Backend {
+	t.Helper()
+	if client == nil {
+		client = newFakeClient(nil)
+	}
+	backend, err := NewWithClient(client, Options{
+		Endpoint:       "https://s3.example.com",
+		Bucket:         "reports",
+		Prefix:         prefix,
+		Region:         DefaultRegion,
+		ForcePathStyle: true,
+	})
+	if err != nil {
+		t.Fatalf("NewWithClient() error = %v", err)
+	}
+	return backend
+}
+
+type fakeClient struct {
+	objects      map[string]string
+	contentTypes map[string]string
+	listPages    []awss3.ListObjectsV2Output
+	tokens       []string
+	putKeys      []string
+	deleteKeys   []string
+}
+
+func newFakeClient(objects map[string]string) *fakeClient {
+	copied := make(map[string]string)
+	for key, value := range objects {
+		copied[key] = value
+	}
+	return &fakeClient{
+		objects:      copied,
+		contentTypes: make(map[string]string),
+	}
+}
+
+func (c *fakeClient) HeadObject(ctx context.Context, input *awss3.HeadObjectInput, optFns ...func(*awss3.Options)) (*awss3.HeadObjectOutput, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	value, ok := c.objects[aws.ToString(input.Key)]
+	if !ok {
+		return nil, &types.NotFound{}
+	}
+	return &awss3.HeadObjectOutput{ContentLength: aws.Int64(int64(len(value)))}, nil
+}
+
+func (c *fakeClient) GetObject(ctx context.Context, input *awss3.GetObjectInput, optFns ...func(*awss3.Options)) (*awss3.GetObjectOutput, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	value, ok := c.objects[aws.ToString(input.Key)]
+	if !ok {
+		return nil, &types.NotFound{}
+	}
+	return &awss3.GetObjectOutput{
+		Body:          io.NopCloser(stringsReader(value)),
+		ContentLength: aws.Int64(int64(len(value))),
+	}, nil
+}
+
+func (c *fakeClient) PutObject(ctx context.Context, input *awss3.PutObjectInput, optFns ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	data, err := io.ReadAll(input.Body)
+	if err != nil {
+		return nil, err
+	}
+	key := aws.ToString(input.Key)
+	c.objects[key] = string(data)
+	c.contentTypes[key] = aws.ToString(input.ContentType)
+	c.putKeys = append(c.putKeys, key)
+	return &awss3.PutObjectOutput{}, nil
+}
+
+func (c *fakeClient) ListObjectsV2(ctx context.Context, input *awss3.ListObjectsV2Input, optFns ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	c.tokens = append(c.tokens, aws.ToString(input.ContinuationToken))
+	if len(c.listPages) > 0 {
+		index := len(c.tokens) - 1
+		if index >= len(c.listPages) {
+			return &awss3.ListObjectsV2Output{IsTruncated: aws.Bool(false)}, nil
+		}
+		page := c.listPages[index]
+		return &page, nil
+	}
+	return c.dynamicList(input), nil
+}
+
+func (c *fakeClient) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput, optFns ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	key := aws.ToString(input.Key)
+	delete(c.objects, key)
+	c.deleteKeys = append(c.deleteKeys, key)
+	return &awss3.DeleteObjectOutput{}, nil
+}
+
+func (c *fakeClient) dynamicList(input *awss3.ListObjectsV2Input) *awss3.ListObjectsV2Output {
+	prefix := aws.ToString(input.Prefix)
+	delimiter := aws.ToString(input.Delimiter)
+	var contents []types.Object
+	commonPrefixes := make(map[string]struct{})
+	for key, value := range c.objects {
+		if !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		remainder := strings.TrimPrefix(key, prefix)
+		if delimiter != "" {
+			if index := strings.Index(remainder, delimiter); index >= 0 {
+				commonPrefixes[prefix+remainder[:index+1]] = struct{}{}
+				continue
+			}
+		}
+		contents = append(contents, types.Object{Key: aws.String(key), Size: aws.Int64(int64(len(value)))})
+	}
+	sort.Slice(contents, func(i, j int) bool { return aws.ToString(contents[i].Key) < aws.ToString(contents[j].Key) })
+	prefixes := make([]types.CommonPrefix, 0, len(commonPrefixes))
+	for prefix := range commonPrefixes {
+		prefixes = append(prefixes, types.CommonPrefix{Prefix: aws.String(prefix)})
+	}
+	sort.Slice(prefixes, func(i, j int) bool { return aws.ToString(prefixes[i].Prefix) < aws.ToString(prefixes[j].Prefix) })
+	return &awss3.ListObjectsV2Output{
+		Contents:       contents,
+		CommonPrefixes: prefixes,
+		IsTruncated:    aws.Bool(false),
+	}
+}
+
+func stringsReader(value string) io.Reader {
+	return strings.NewReader(value)
+}
+
+func entryPaths(entries []storage.Entry) []string {
+	paths := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		paths = append(paths, entry.Path)
+	}
+	return paths
+}
+
+func sortedStrings(values []string) []string {
+	copied := append([]string(nil), values...)
+	sort.Strings(copied)
+	return copied
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for index := range a {
+		if a[index] != b[index] {
+			return false
+		}
+	}
+	return true
+}

internal/adapters/s3/integration_test.go

@@ -0,0 +1,39 @@
+package s3
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"testing"
+)
+
+func TestIntegrationS3BackendHasAny(t *testing.T) {
+	endpoint := os.Getenv("DISTRIBUTOR_TEST_S3_ENDPOINT")
+	bucket := os.Getenv("DISTRIBUTOR_TEST_S3_BUCKET")
+	if endpoint == "" || bucket == "" {
+		t.Skip("DISTRIBUTOR_TEST_S3_ENDPOINT and DISTRIBUTOR_TEST_S3_BUCKET are not set")
+	}
+	forcePathStyle := true
+	if raw := os.Getenv("DISTRIBUTOR_TEST_S3_FORCE_PATH_STYLE"); raw != "" {
+		parsed, err := strconv.ParseBool(raw)
+		if err != nil {
+			t.Fatalf("parse DISTRIBUTOR_TEST_S3_FORCE_PATH_STYLE: %v", err)
+		}
+		forcePathStyle = parsed
+	}
+	backend, err := New(context.Background(), Options{
+		Endpoint:        endpoint,
+		Bucket:          bucket,
+		Prefix:          os.Getenv("DISTRIBUTOR_TEST_S3_PREFIX"),
+		Region:          os.Getenv("DISTRIBUTOR_TEST_S3_REGION"),
+		ForcePathStyle:  forcePathStyle,
+		AccessKeyID:     os.Getenv("DISTRIBUTOR_TEST_S3_ACCESS_KEY_ID"),
+		SecretAccessKey: os.Getenv("DISTRIBUTOR_TEST_S3_SECRET_ACCESS_KEY"),
+	})
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	if _, err := backend.HasAny(context.Background(), ""); err != nil {
+		t.Fatalf("HasAny(root) error = %v", err)
+	}
+}

internal/adapters/s3/options.go

@@ -0,0 +1,42 @@
+package s3
+
+import (
+	"fmt"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+const BackendName = "s3"
+
+const DefaultRegion = "us-east-1"
+
+type Options struct {
+	Endpoint        string
+	Bucket          string
+	Prefix          string
+	Region          string
+	ForcePathStyle  bool
+	AccessKeyID     string
+	SecretAccessKey string
+}
+
+func (o Options) normalized() (Options, error) {
+	if o.Endpoint == "" {
+		return Options{}, fmt.Errorf("endpoint is required")
+	}
+	if o.Bucket == "" {
+		return Options{}, fmt.Errorf("bucket is required")
+	}
+	if o.Region == "" {
+		o.Region = DefaultRegion
+	}
+	o.Prefix = strings.Trim(o.Prefix, "/")
+	if err := storage.ValidatePrefix(o.Prefix); err != nil {
+		return Options{}, fmt.Errorf("prefix: %w", err)
+	}
+	if (o.AccessKeyID == "") != (o.SecretAccessKey == "") {
+		return Options{}, fmt.Errorf("access key id and secret access key must be configured together")
+	}
+	return o, nil
+}

internal/adapters/ssh/auth.go

@@ -0,0 +1,61 @@
+package ssh
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+
+	cryptossh "golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
+)
+
+type agentDialer func(network, address string) (net.Conn, error)
+
+func authMethods(keyFile string) ([]cryptossh.AuthMethod, func(), error) {
+	return authMethodsWithAgent(os.Getenv("SSH_AUTH_SOCK"), net.Dial, keyFile)
+}
+
+func authMethodsWithAgent(agentSocket string, dial agentDialer, keyFile string) ([]cryptossh.AuthMethod, func(), error) {
+	var methods []cryptossh.AuthMethod
+	var closers []io.Closer
+	if agentSocket != "" {
+		methods = append(methods, cryptossh.PublicKeysCallback(func() ([]cryptossh.Signer, error) {
+			conn, err := dial("unix", agentSocket)
+			if err != nil {
+				return nil, err
+			}
+			closers = append(closers, conn)
+			return agent.NewClient(conn).Signers()
+		}))
+	}
+	if keyFile != "" {
+		signer, err := signerFromKeyFile(keyFile)
+		if err != nil {
+			return nil, nil, err
+		}
+		methods = append(methods, cryptossh.PublicKeys(signer))
+	}
+	if len(methods) == 0 {
+		return nil, nil, fmt.Errorf("no SSH auth methods configured; set SSH_AUTH_SOCK or ssh_key_file")
+	}
+	return methods, func() { closeAll(closers) }, nil
+}
+
+func signerFromKeyFile(path string) (cryptossh.Signer, error) {
+	key, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read ssh_key_file %q: %w", path, err)
+	}
+	signer, err := cryptossh.ParsePrivateKey(key)
+	if err != nil {
+		return nil, fmt.Errorf("parse ssh_key_file %q: %w", path, err)
+	}
+	return signer, nil
+}
+
+func closeAll(closers []io.Closer) {
+	for _, closer := range closers {
+		_ = closer.Close()
+	}
+}

internal/adapters/ssh/auth_test.go

@@ -0,0 +1,59 @@
+package ssh
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestAuthMethodsPreferAgentBeforeKeyFile(t *testing.T) {
+	keyFile := writePrivateKey(t)
+	methods, cleanup, err := authMethodsWithAgent("/tmp/ssh-agent.sock", nil, keyFile)
+	if err != nil {
+		t.Fatalf("authMethodsWithAgent() error = %v", err)
+	}
+	defer cleanup()
+	if got, want := len(methods), 2; got != want {
+		t.Fatalf("auth method count = %d, want %d", got, want)
+	}
+}
+
+func TestAuthMethodsLoadsKeyFile(t *testing.T) {
+	keyFile := writePrivateKey(t)
+	methods, cleanup, err := authMethodsWithAgent("", nil, keyFile)
+	if err != nil {
+		t.Fatalf("authMethodsWithAgent() error = %v", err)
+	}
+	defer cleanup()
+	if got, want := len(methods), 1; got != want {
+		t.Fatalf("auth method count = %d, want %d", got, want)
+	}
+}
+
+func TestAuthMethodsRejectsMissingAuth(t *testing.T) {
+	_, _, err := authMethodsWithAgent("", nil, "")
+	if err == nil {
+		t.Fatal("authMethodsWithAgent() error = nil, want error")
+	}
+}
+
+func writePrivateKey(t *testing.T) string {
+	t.Helper()
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("generate key: %v", err)
+	}
+	data := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
+	})
+	path := filepath.Join(t.TempDir(), "id_rsa")
+	if err := os.WriteFile(path, data, 0o600); err != nil {
+		t.Fatalf("write private key: %v", err)
+	}
+	return path
+}

internal/adapters/ssh/backend.go

@@ -0,0 +1,487 @@
+package ssh
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"github.com/pkg/sftp"
+	cryptossh "golang.org/x/crypto/ssh"
+)
+
+type Backend struct {
+	client    *sftp.Client
+	sshClient *cryptossh.Client
+	root      string
+}
+
+func New(ctx context.Context, options Options) (*Backend, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	options, err := options.normalized()
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.Root, storage.ErrInvalidPath, err)
+	}
+	hostKeyCallback, err := hostKeyCallback(options)
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.KnownHosts, storage.ErrInvalidPath, err)
+	}
+	auth, cleanupAuth, err := authMethods(options.KeyFile)
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.KeyFile, storage.ErrPermission, err)
+	}
+	defer cleanupAuth()
+
+	sshClient, err := cryptossh.Dial("tcp", options.address(), &cryptossh.ClientConfig{
+		User:            options.User,
+		Auth:            auth,
+		HostKeyCallback: hostKeyCallback,
+		Timeout:         30 * time.Second,
+	})
+	if err != nil {
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.address(), storage.ErrUnknown, err)
+	}
+	client, err := sftp.NewClient(sshClient)
+	if err != nil {
+		_ = sshClient.Close()
+		return nil, storage.NewError(storage.OpOpenBackend, BackendName, options.address(), storage.ErrUnknown, err)
+	}
+	return &Backend{client: client, sshClient: sshClient, root: options.Root}, nil
+}
+
+func (b *Backend) Close() error {
+	var err error
+	if b.client != nil {
+		err = b.client.Close()
+	}
+	if b.sshClient != nil {
+		if closeErr := b.sshClient.Close(); err == nil {
+			err = closeErr
+		}
+	}
+	return err
+}
+
+func (b *Backend) ReadFile(ctx context.Context, logicalPath string) ([]byte, error) {
+	reader, err := b.OpenReader(ctx, logicalPath)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, storage.NewError(storage.OpReadFile, BackendName, logicalPath, storage.ErrUnknown, err)
+	}
+	return data, nil
+}
+
+func (b *Backend) OpenReader(ctx context.Context, logicalPath string) (io.ReadCloser, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	nativePath, err := b.nativePath(logicalPath, false)
+	if err != nil {
+		return nil, err
+	}
+	if err := b.rejectSymlinkAncestors(ctx, logicalPath, true); err != nil {
+		return nil, err
+	}
+	info, err := b.client.Lstat(nativePath)
+	if err != nil {
+		return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
+	}
+	if !info.Mode().IsRegular() {
+		return nil, storage.NewError(storage.OpOpenReader, BackendName, logicalPath, storage.ErrUnsupported, nil)
+	}
+	file, err := b.client.Open(nativePath)
+	if err != nil {
+		return nil, b.translateError(storage.OpOpenReader, logicalPath, err)
+	}
+	return file, nil
+}
+
+func (b *Backend) WriteFile(ctx context.Context, logicalPath string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
+	opts.Size = int64(len(data))
+	opts.SizeKnown = true
+	return b.WriteFrom(ctx, logicalPath, bytes.NewReader(data), opts)
+}
+
+func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	nativePath, err := b.nativePath(logicalPath, false)
+	if err != nil {
+		return storage.Entry{}, err
+	}
+	if err := b.rejectSymlinkAncestors(ctx, parentOf(logicalPath), true); err != nil {
+		return storage.Entry{}, err
+	}
+	if info, err := b.client.Lstat(nativePath); err == nil {
+		if !opts.Overwrite {
+			return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrAlreadyExist, nil)
+		}
+		if !info.Mode().IsRegular() {
+			return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, nil)
+		}
+	} else if !isNotExist(err) {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
+	}
+
+	parentNative := path.Dir(nativePath)
+	if err := b.client.MkdirAll(parentNative); err != nil {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
+	}
+
+	writePath := nativePath
+	if opts.PreferAtomic {
+		writePath = path.Join(parentNative, fmt.Sprintf(".distributor-write-%d", time.Now().UnixNano()))
+	}
+	file, err := b.client.Create(writePath)
+	if err != nil {
+		return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
+	}
+	cleanup := opts.PreferAtomic
+	defer func() {
+		if cleanup {
+			_ = b.client.Remove(writePath)
+		}
+	}()
+
+	written, copyErr := io.Copy(file, r)
+	closeErr := file.Close()
+	if copyErr != nil {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, copyErr)
+	}
+	if closeErr != nil {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrUnknown, closeErr)
+	}
+	if opts.SizeKnown && written != opts.Size {
+		return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", written, opts.Size))
+	}
+	if opts.PreferAtomic {
+		if err := b.client.Rename(writePath, nativePath); err != nil {
+			return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
+		}
+		cleanup = false
+	}
+	return b.Stat(ctx, logicalPath)
+}
+
+func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) {
+	if err := ctx.Err(); err != nil {
+		return storage.Entry{}, err
+	}
+	nativePath, err := b.nativePath(logicalPath, true)
+	if err != nil {
+		return storage.Entry{}, err
+	}
+	info, err := b.client.Lstat(nativePath)
+	if err != nil {
+		return storage.Entry{}, b.translateError(storage.OpStat, logicalPath, err)
+	}
+	return entryFromInfo(logicalPath, info), nil
+}
+
+func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	nativePrefix, err := b.nativePath(prefix, true)
+	if err != nil {
+		return err
+	}
+	info, err := b.client.Lstat(nativePrefix)
+	if err != nil {
+		if isNotExist(err) {
+			return nil
+		}
+		return b.translateError(storage.OpWalk, prefix, err)
+	}
+
+	emitter := storage.NewWalkEmitter(ctx, BackendName, opts, fn)
+
+	if !info.IsDir() {
+		return storage.FinishWalk(emitter.Emit(entryFromInfo(prefix, info)))
+	}
+
+	return storage.FinishWalk(b.walkDirectory(ctx, prefix, nativePrefix, opts, emitter.Emit))
+}
+
+func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
+	return storage.HasAny(ctx, b, prefix)
+}
+
+func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
+	if err != nil {
+		return err
+	}
+	for _, target := range targets {
+		nativePath, err := b.nativePath(target, false)
+		if err != nil {
+			return err
+		}
+		if nativePath == b.root {
+			return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrInvalidPath, nil)
+		}
+		info, err := b.client.Lstat(nativePath)
+		if err != nil {
+			if opts.IgnoreMissing && isNotExist(err) {
+				continue
+			}
+			return b.translateError(storage.OpDeleteManagedBundle, target, err)
+		}
+		if info.IsDir() {
+			return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrUnsupported, nil)
+		}
+		if err := b.client.Remove(nativePath); err != nil {
+			return b.translateError(storage.OpDeleteManagedBundle, target, err)
+		}
+		if opts.PruneEmptyDirs {
+			b.pruneEmptyParents(parentOf(target))
+		}
+	}
+	return nil
+}
+
+func (b *Backend) DeletePrefix(ctx context.Context, prefix string, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if err := storage.ValidatePrefix(prefix); err != nil {
+		return err
+	}
+	var entries []storage.Entry
+	if prefix != "" {
+		entry, err := b.Stat(ctx, prefix)
+		if err != nil {
+			if opts.IgnoreMissing && storage.IsNotFound(err) {
+				return nil
+			}
+			return err
+		}
+		if entry.Type != storage.EntryTypeDirectory {
+			return b.deleteEntry(ctx, entry, opts)
+		}
+		entries = append(entries, entry)
+	}
+	if err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: true}, func(entry storage.Entry) error {
+		entries = append(entries, entry)
+		return nil
+	}); err != nil {
+		return err
+	}
+	if prefix != "" && len(entries) == 1 {
+		if err := b.deleteEntry(ctx, entries[0], opts); err != nil {
+			return err
+		}
+		if opts.PruneEmptyDirs {
+			b.pruneEmptyParents(parentOf(prefix))
+		}
+		return nil
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return strings.Count(entries[i].Path, "/") > strings.Count(entries[j].Path, "/")
+	})
+	for _, entry := range entries {
+		if entry.Path == "" {
+			continue
+		}
+		if err := b.deleteEntry(ctx, entry, storage.DeleteOptions{IgnoreMissing: true}); err != nil {
+			return err
+		}
+	}
+	if opts.PruneEmptyDirs {
+		b.pruneEmptyParents(parentOf(prefix))
+	}
+	return nil
+}
+
+func (b *Backend) deleteEntry(ctx context.Context, entry storage.Entry, opts storage.DeleteOptions) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	nativePath, err := b.nativePath(entry.Path, false)
+	if err != nil {
+		return err
+	}
+	var removeErr error
+	if entry.Type == storage.EntryTypeDirectory {
+		removeErr = b.client.RemoveDirectory(nativePath)
+	} else {
+		removeErr = b.client.Remove(nativePath)
+	}
+	if removeErr != nil {
+		if opts.IgnoreMissing && isNotExist(removeErr) {
+			return nil
+		}
+		return b.translateError(storage.OpDeletePrefix, entry.Path, removeErr)
+	}
+	return nil
+}
+
+func (b *Backend) walkDirectory(ctx context.Context, logicalPrefix, nativePrefix string, opts storage.WalkOptions, emit func(storage.Entry) error) error {
+	entries, err := b.client.ReadDir(nativePrefix)
+	if err != nil {
+		return b.translateError(storage.OpWalk, logicalPrefix, err)
+	}
+	sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })
+	for _, info := range entries {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		logicalPath := info.Name()
+		if logicalPrefix != "" {
+			logicalPath = logicalPrefix + "/" + info.Name()
+		}
+		if err := emit(entryFromInfo(logicalPath, info)); err != nil {
+			return err
+		}
+		if opts.Recursive && info.IsDir() {
+			if err := b.walkDirectory(ctx, logicalPath, path.Join(nativePrefix, info.Name()), opts, emit); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) {
+	if logicalPath == "" {
+		if !allowEmpty {
+			return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
+		}
+		return b.root, nil
+	}
+	if err := storage.ValidatePath(logicalPath); err != nil {
+		return "", err
+	}
+	nativePath := path.Clean(path.Join(b.root, logicalPath))
+	if !withinRoot(b.root, nativePath) {
+		return "", storage.NewError(storage.OpValidatePath, BackendName, logicalPath, storage.ErrInvalidPath, nil)
+	}
+	return nativePath, nil
+}
+
+func (b *Backend) rejectSymlinkAncestors(ctx context.Context, logicalPath string, includeFinal bool) error {
+	if logicalPath == "" {
+		return nil
+	}
+	if err := storage.ValidatePath(logicalPath); err != nil {
+		return err
+	}
+	segments := strings.Split(logicalPath, "/")
+	limit := len(segments)
+	if !includeFinal {
+		limit--
+	}
+	current := ""
+	for index := 0; index < limit; index++ {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if current == "" {
+			current = segments[index]
+		} else {
+			current += "/" + segments[index]
+		}
+		nativePath, err := b.nativePath(current, false)
+		if err != nil {
+			return err
+		}
+		info, err := b.client.Lstat(nativePath)
+		if err != nil {
+			if isNotExist(err) {
+				return nil
+			}
+			return b.translateError(storage.OpStat, current, err)
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			return storage.NewError(storage.OpStat, BackendName, current, storage.ErrUnsupported, nil)
+		}
+	}
+	return nil
+}
+
+func (b *Backend) pruneEmptyParents(logicalPath string) {
+	for logicalPath != "" {
+		nativePath, err := b.nativePath(logicalPath, false)
+		if err != nil || nativePath == b.root {
+			return
+		}
+		if err := b.client.RemoveDirectory(nativePath); err != nil {
+			return
+		}
+		logicalPath = parentOf(logicalPath)
+	}
+}
+
+func withinRoot(root, candidate string) bool {
+	if candidate == root {
+		return true
+	}
+	if root == "/" {
+		return strings.HasPrefix(candidate, "/")
+	}
+	return strings.HasPrefix(candidate, strings.TrimSuffix(root, "/")+"/")
+}
+
+func parentOf(logicalPath string) string {
+	index := strings.LastIndex(logicalPath, "/")
+	if index == -1 {
+		return ""
+	}
+	return logicalPath[:index]
+}
+
+func isNotExist(err error) bool {
+	return errors.Is(err, fs.ErrNotExist) || errors.Is(err, os.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile)
+}
+
+func (b *Backend) translateError(op, logicalPath string, err error) error {
+	kind := storage.ErrUnknown
+	switch {
+	case isNotExist(err):
+		kind = storage.ErrNotFound
+	case errors.Is(err, fs.ErrExist), errors.Is(err, os.ErrExist):
+		kind = storage.ErrAlreadyExist
+	case errors.Is(err, fs.ErrPermission), errors.Is(err, os.ErrPermission), errors.Is(err, sftp.ErrSSHFxPermissionDenied):
+		kind = storage.ErrPermission
+	case errors.Is(err, sftp.ErrSSHFxOpUnsupported):
+		kind = storage.ErrUnsupported
+	case errors.Is(err, sftp.ErrSSHFxNoConnection), errors.Is(err, sftp.ErrSSHFxConnectionLost):
+		kind = storage.ErrTemporary
+	}
+	return storage.NewError(op, BackendName, logicalPath, kind, err)
+}
+
+func entryFromInfo(logicalPath string, info fs.FileInfo) storage.Entry {
+	entryType := storage.EntryTypeOther
+	switch {
+	case info.Mode()&os.ModeSymlink != 0:
+		entryType = storage.EntryTypeSymlink
+	case info.Mode().IsRegular():
+		entryType = storage.EntryTypeFile
+	case info.IsDir():
+		entryType = storage.EntryTypeDirectory
+	}
+	return storage.Entry{
+		Path: logicalPath,
+		Type: entryType,
+		Size: info.Size(),
+	}
+}

internal/adapters/ssh/hostkeys.go

@@ -0,0 +1,84 @@
+package ssh
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"os"
+
+	cryptossh "golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+)
+
+func hostKeyCallback(options Options) (cryptossh.HostKeyCallback, error) {
+	switch options.HostKeyPolicy {
+	case HostKeyPolicyOff:
+		return cryptossh.InsecureIgnoreHostKey(), nil
+	case HostKeyPolicyStrict:
+		if options.KnownHosts == "" {
+			return nil, fmt.Errorf("known_hosts is required for strict host key checking")
+		}
+		callback, err := knownhosts.New(options.KnownHosts)
+		if err != nil {
+			return nil, fmt.Errorf("load known_hosts %q: %w", options.KnownHosts, err)
+		}
+		return callback, nil
+	case HostKeyPolicyAcceptNew:
+		return acceptNewHostKeyCallback(options)
+	default:
+		return nil, fmt.Errorf("host_key_policy must be strict, accept-new, or off")
+	}
+}
+
+func acceptNewHostKeyCallback(options Options) (cryptossh.HostKeyCallback, error) {
+	var checker cryptossh.HostKeyCallback
+	if options.KnownHosts != "" {
+		loaded, err := knownhosts.New(options.KnownHosts)
+		if err == nil {
+			checker = loaded
+		} else if !errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("load known_hosts %q: %w", options.KnownHosts, err)
+		}
+	}
+	return func(hostname string, remote net.Addr, key cryptossh.PublicKey) error {
+		if checker != nil {
+			err := checker(hostname, remote, key)
+			if err == nil {
+				return nil
+			}
+			var keyErr *knownhosts.KeyError
+			if !errors.As(err, &keyErr) {
+				return err
+			}
+			if len(keyErr.Want) > 0 {
+				return fmt.Errorf("host key for %s has changed: %w", hostname, err)
+			}
+		}
+		if options.ReadOnlyKnownHosts {
+			return nil
+		}
+		if options.KnownHosts == "" {
+			return fmt.Errorf("host key for %s is unknown and no writable known_hosts path is available", hostname)
+		}
+		if err := appendKnownHost(options.KnownHosts, hostname, key); err != nil {
+			return err
+		}
+		loaded, err := knownhosts.New(options.KnownHosts)
+		if err == nil {
+			checker = loaded
+		}
+		return nil
+	}, nil
+}
+
+func appendKnownHost(path, host string, key cryptossh.PublicKey) error {
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+	if err != nil {
+		return fmt.Errorf("persist accepted host key to known_hosts %q: %w", path, err)
+	}
+	defer file.Close()
+	if _, err := fmt.Fprintln(file, knownhosts.Line([]string{knownhosts.Normalize(host)}, key)); err != nil {
+		return fmt.Errorf("persist accepted host key to known_hosts %q: %w", path, err)
+	}
+	return nil
+}

internal/adapters/ssh/hostkeys_test.go

@@ -0,0 +1,138 @@
+package ssh
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	cryptossh "golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+)
+
+func TestAcceptNewHostKeyCallbackPersistsUnknownHost(t *testing.T) {
+	key := testPublicKey(t)
+	knownHosts := filepath.Join(t.TempDir(), "known_hosts")
+	callback, err := acceptNewHostKeyCallback(Options{KnownHosts: knownHosts})
+	if err != nil {
+		t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
+	}
+
+	if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil {
+		t.Fatalf("callback() error = %v", err)
+	}
+	data, err := os.ReadFile(knownHosts)
+	if err != nil {
+		t.Fatalf("read known_hosts: %v", err)
+	}
+	if !strings.Contains(string(data), "example.com") {
+		t.Fatalf("known_hosts = %q, want example.com entry", data)
+	}
+	if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil {
+		t.Fatalf("second callback() error = %v", err)
+	}
+}
+
+func TestAcceptNewHostKeyCallbackReadOnlyDoesNotPersistUnknownHost(t *testing.T) {
+	key := testPublicKey(t)
+	knownHosts := filepath.Join(t.TempDir(), "known_hosts")
+	callback, err := acceptNewHostKeyCallback(Options{
+		KnownHosts:         knownHosts,
+		ReadOnlyKnownHosts: true,
+	})
+	if err != nil {
+		t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
+	}
+
+	if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, key); err != nil {
+		t.Fatalf("callback() error = %v", err)
+	}
+	if _, err := os.Stat(knownHosts); !os.IsNotExist(err) {
+		t.Fatalf("known_hosts stat error = %v, want not exist", err)
+	}
+}
+
+func TestAcceptNewHostKeyCallbackRejectsChangedHostKey(t *testing.T) {
+	first := testPublicKey(t)
+	second := testPublicKey(t)
+	knownHosts := filepath.Join(t.TempDir(), "known_hosts")
+	if err := os.WriteFile(knownHosts, []byte(knownhosts.Line([]string{knownhosts.Normalize("example.com:22")}, first)+"\n"), 0o600); err != nil {
+		t.Fatalf("write known_hosts: %v", err)
+	}
+	callback, err := acceptNewHostKeyCallback(Options{KnownHosts: knownHosts})
+	if err != nil {
+		t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
+	}
+
+	err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, second)
+	if err == nil || !strings.Contains(err.Error(), "has changed") {
+		t.Fatalf("callback() error = %v, want changed host key", err)
+	}
+}
+
+func TestAcceptNewHostKeyCallbackReadOnlyRejectsChangedHostKey(t *testing.T) {
+	first := testPublicKey(t)
+	second := testPublicKey(t)
+	knownHosts := filepath.Join(t.TempDir(), "known_hosts")
+	if err := os.WriteFile(knownHosts, []byte(knownhosts.Line([]string{knownhosts.Normalize("example.com:22")}, first)+"\n"), 0o600); err != nil {
+		t.Fatalf("write known_hosts: %v", err)
+	}
+	callback, err := acceptNewHostKeyCallback(Options{
+		KnownHosts:         knownHosts,
+		ReadOnlyKnownHosts: true,
+	})
+	if err != nil {
+		t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
+	}
+
+	err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, second)
+	if err == nil || !strings.Contains(err.Error(), "has changed") {
+		t.Fatalf("callback() error = %v, want changed host key", err)
+	}
+}
+
+func TestAcceptNewHostKeyCallbackRequiresWritableKnownHostsForUnknownHost(t *testing.T) {
+	callback, err := acceptNewHostKeyCallback(Options{})
+	if err != nil {
+		t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
+	}
+
+	err = callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, testPublicKey(t))
+	if err == nil || !strings.Contains(err.Error(), "no writable known_hosts path") {
+		t.Fatalf("callback() error = %v, want no writable known_hosts path", err)
+	}
+}
+
+func TestAcceptNewHostKeyCallbackReadOnlyAllowsMissingKnownHosts(t *testing.T) {
+	callback, err := acceptNewHostKeyCallback(Options{ReadOnlyKnownHosts: true})
+	if err != nil {
+		t.Fatalf("acceptNewHostKeyCallback() error = %v", err)
+	}
+
+	if err := callback("example.com:22", &net.TCPAddr{IP: net.ParseIP("192.0.2.10"), Port: 22}, testPublicKey(t)); err != nil {
+		t.Fatalf("callback() error = %v", err)
+	}
+}
+
+func TestStrictHostKeyCallbackRequiresKnownHosts(t *testing.T) {
+	_, err := hostKeyCallback(Options{HostKeyPolicy: HostKeyPolicyStrict})
+	if err == nil || !strings.Contains(err.Error(), "known_hosts is required") {
+		t.Fatalf("hostKeyCallback() error = %v, want known_hosts required", err)
+	}
+}
+
+func testPublicKey(t *testing.T) cryptossh.PublicKey {
+	t.Helper()
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("generate key: %v", err)
+	}
+	publicKey, err := cryptossh.NewPublicKey(&privateKey.PublicKey)
+	if err != nil {
+		t.Fatalf("new public key: %v", err)
+	}
+	return publicKey
+}

internal/adapters/ssh/integration_test.go

@@ -0,0 +1,39 @@
+package ssh
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"testing"
+)
+
+func TestIntegrationSSHBackendStatRoot(t *testing.T) {
+	host := os.Getenv("DISTRIBUTOR_TEST_SSH_HOST")
+	if host == "" {
+		t.Skip("DISTRIBUTOR_TEST_SSH_HOST is not set")
+	}
+	port := 22
+	if raw := os.Getenv("DISTRIBUTOR_TEST_SSH_PORT"); raw != "" {
+		parsed, err := strconv.Atoi(raw)
+		if err != nil {
+			t.Fatalf("parse DISTRIBUTOR_TEST_SSH_PORT: %v", err)
+		}
+		port = parsed
+	}
+	backend, err := New(context.Background(), Options{
+		Host:          host,
+		User:          os.Getenv("DISTRIBUTOR_TEST_SSH_USER"),
+		Port:          port,
+		Root:          os.Getenv("DISTRIBUTOR_TEST_SSH_PATH"),
+		KeyFile:       os.Getenv("DISTRIBUTOR_TEST_SSH_KEY_FILE"),
+		KnownHosts:    os.Getenv("DISTRIBUTOR_TEST_SSH_KNOWN_HOSTS"),
+		HostKeyPolicy: HostKeyPolicyStrict,
+	})
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	defer backend.Close()
+	if _, err := backend.Stat(context.Background(), ""); err != nil {
+		t.Fatalf("Stat(root) error = %v", err)
+	}
+}

internal/adapters/ssh/options.go

@@ -0,0 +1,78 @@
+package ssh
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"path"
+	"path/filepath"
+	"strconv"
+)
+
+const (
+	BackendName = "ssh"
+
+	HostKeyPolicyStrict    HostKeyPolicy = "strict"
+	HostKeyPolicyAcceptNew HostKeyPolicy = "accept-new"
+	HostKeyPolicyOff       HostKeyPolicy = "off"
+)
+
+type HostKeyPolicy string
+
+type Options struct {
+	Host               string
+	User               string
+	Port               int
+	Root               string
+	KeyFile            string
+	KnownHosts         string
+	HostKeyPolicy      HostKeyPolicy
+	ReadOnlyKnownHosts bool
+}
+
+func (o Options) normalized() (Options, error) {
+	if o.Host == "" {
+		return Options{}, fmt.Errorf("host is required")
+	}
+	if o.User == "" {
+		current, err := user.Current()
+		if err != nil || current.Username == "" {
+			return Options{}, fmt.Errorf("user is required when current OS user cannot be determined")
+		}
+		o.User = current.Username
+	}
+	if o.Port == 0 {
+		o.Port = 22
+	}
+	if o.Port < 1 || o.Port > 65535 {
+		return Options{}, fmt.Errorf("port must be between 1 and 65535")
+	}
+	if o.Root == "" {
+		return Options{}, fmt.Errorf("path is required")
+	}
+	o.Root = path.Clean(o.Root)
+	if o.HostKeyPolicy == "" {
+		o.HostKeyPolicy = HostKeyPolicyAcceptNew
+	}
+	switch o.HostKeyPolicy {
+	case HostKeyPolicyStrict, HostKeyPolicyAcceptNew, HostKeyPolicyOff:
+	default:
+		return Options{}, fmt.Errorf("host_key_policy must be strict, accept-new, or off")
+	}
+	if o.KnownHosts == "" && o.HostKeyPolicy != HostKeyPolicyOff {
+		o.KnownHosts = defaultKnownHostsPath()
+	}
+	return o, nil
+}
+
+func (o Options) address() string {
+	return o.Host + ":" + strconv.Itoa(o.Port)
+}
+
+func defaultKnownHostsPath() string {
+	home, err := os.UserHomeDir()
+	if err != nil || home == "" {
+		return ""
+	}
+	return filepath.Join(home, ".ssh", "known_hosts")
+}

internal/adapters/ssh/options_test.go

@@ -0,0 +1,118 @@
+package ssh
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"github.com/pkg/sftp"
+)
+
+func TestOptionsNormalizeDefaultsUserPortAndHostKeyPolicy(t *testing.T) {
+	options, err := (Options{
+		Host: "example.com",
+		Root: "/reports",
+	}).normalized()
+	if err != nil {
+		t.Fatalf("normalized() error = %v", err)
+	}
+	if options.User == "" {
+		t.Fatal("normalized user is empty")
+	}
+	if options.Port != 22 {
+		t.Fatalf("port = %d, want 22", options.Port)
+	}
+	if options.HostKeyPolicy != HostKeyPolicyAcceptNew {
+		t.Fatalf("host key policy = %q, want accept-new", options.HostKeyPolicy)
+	}
+}
+
+func TestOptionsNormalizeRejectsInvalidFields(t *testing.T) {
+	tests := map[string]Options{
+		"host": {Root: "/reports"},
+		"port": {
+			Host: "example.com",
+			Port: 70000,
+			Root: "/reports",
+		},
+		"path": {
+			Host: "example.com",
+		},
+		"host key policy": {
+			Host:          "example.com",
+			Root:          "/reports",
+			HostKeyPolicy: "prompt",
+		},
+	}
+	for name, options := range tests {
+		t.Run(name, func(t *testing.T) {
+			if _, err := options.normalized(); err == nil {
+				t.Fatal("normalized() error = nil, want error")
+			}
+		})
+	}
+}
+
+func TestNativePathEnforcesLogicalPathRules(t *testing.T) {
+	backend := &Backend{root: "/srv/reports"}
+	tests := map[string]string{
+		"bundle/report.md": "/srv/reports/bundle/report.md",
+		"":                 "/srv/reports",
+	}
+	for logicalPath, want := range tests {
+		t.Run(logicalPath, func(t *testing.T) {
+			got, err := backend.nativePath(logicalPath, true)
+			if err != nil {
+				t.Fatalf("nativePath() error = %v", err)
+			}
+			if got != want {
+				t.Fatalf("nativePath() = %q, want %q", got, want)
+			}
+		})
+	}
+
+	for _, logicalPath := range []string{"/absolute", "../escape", "a/../b", `a\b`} {
+		t.Run("reject "+logicalPath, func(t *testing.T) {
+			_, err := backend.nativePath(logicalPath, true)
+			if err == nil || !storage.IsInvalidPath(err) {
+				t.Fatalf("nativePath() error = %v, want invalid path", err)
+			}
+		})
+	}
+}
+
+func TestNewRejectsMissingAuthBeforeDial(t *testing.T) {
+	t.Setenv("SSH_AUTH_SOCK", "")
+	_, err := New(context.Background(), Options{
+		Host:          "example.com",
+		User:          "reports",
+		Root:          "/reports",
+		HostKeyPolicy: HostKeyPolicyOff,
+	})
+	if err == nil || !strings.Contains(err.Error(), "no SSH auth methods configured") {
+		t.Fatalf("New() error = %v, want missing auth", err)
+	}
+}
+
+func TestTranslateErrorMapsSFTPStatusCodes(t *testing.T) {
+	backend := &Backend{}
+	tests := []struct {
+		name string
+		err  error
+		want storage.ErrorKind
+	}{
+		{name: "not found", err: sftp.ErrSSHFxNoSuchFile, want: storage.ErrNotFound},
+		{name: "permission", err: sftp.ErrSSHFxPermissionDenied, want: storage.ErrPermission},
+		{name: "unsupported", err: sftp.ErrSSHFxOpUnsupported, want: storage.ErrUnsupported},
+		{name: "temporary", err: sftp.ErrSSHFxConnectionLost, want: storage.ErrTemporary},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := backend.translateError(storage.OpStat, "report.md", tt.err)
+			if !storage.IsKind(err, tt.want) {
+				t.Fatalf("translateError() = %v, want kind %s", err, tt.want)
+			}
+		})
+	}
+}

internal/app/backends.go

@@ -3,19 +3,61 @@ package app
 import (
 	"context"
 	"fmt"
+	"strconv"
 
 	"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
+	s3adapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/s3"
+	sshadapter "gitea.maximumdirect.net/eric/distributor/internal/adapters/ssh"
 	"gitea.maximumdirect.net/eric/distributor/internal/config"
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 )
 
 const storagePathKey = "path"
 
+const (
+	sshHostKey          = "host"
+	sshUserKey          = "user"
+	sshPortKey          = "port"
+	sshKeyFileKey       = "ssh_key_file"
+	sshKnownHostsKey    = "known_hosts"
+	sshHostKeyPolicyKey = "host_key_policy"
+	sshReadOnlyHostsKey = "read_only_known_hosts"
+	s3EndpointKey       = "endpoint"
+	s3BucketKey         = "bucket"
+	s3PrefixKey         = "prefix"
+	s3RegionKey         = "region"
+	s3ForcePathStyleKey = "force_path_style"
+	s3AccessKeyIDKey    = "access_key_id"
+	s3SecretAccessKey   = "secret_access_key"
+)
+
 type backendFactory struct {
-	registry *storage.Registry
+	registry           *storage.Registry
+	environment        config.Environment
+	readOnlyKnownHosts bool
+}
+
+type backendOpenSpec struct {
+	role        string
+	backend     string
+	path        string
+	host        string
+	user        string
+	port        int
+	ssh         config.SSH
+	endpoint    string
+	bucket      string
+	prefix      string
+	region      string
+	forcePath   *bool
+	credentials config.Credentials
 }
 
 func newBackendFactory() *backendFactory {
+	return newBackendFactoryWithEnvironment(config.ProcessEnvironment())
+}
+
+func newBackendFactoryWithEnvironment(environment config.Environment) *backendFactory {
 	registry := storage.NewRegistry()
 	_ = registry.Register(config.BackendLocal, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
 		if err := ctx.Err(); err != nil {
@@ -23,23 +65,149 @@ func newBackendFactory() *backendFactory {
 		}
 		return local.New(cfg[storagePathKey])
 	})
-	return &backendFactory{registry: registry}
+	_ = registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		port, err := strconv.Atoi(cfg[sshPortKey])
+		if err != nil {
+			return nil, fmt.Errorf("ssh port: %w", err)
+		}
+		readOnlyKnownHosts := false
+		if raw := cfg[sshReadOnlyHostsKey]; raw != "" {
+			readOnlyKnownHosts, err = strconv.ParseBool(raw)
+			if err != nil {
+				return nil, fmt.Errorf("ssh read_only_known_hosts: %w", err)
+			}
+		}
+		return sshadapter.New(ctx, sshadapter.Options{
+			Host:               cfg[sshHostKey],
+			User:               cfg[sshUserKey],
+			Port:               port,
+			Root:               cfg[storagePathKey],
+			KeyFile:            cfg[sshKeyFileKey],
+			KnownHosts:         cfg[sshKnownHostsKey],
+			HostKeyPolicy:      sshadapter.HostKeyPolicy(cfg[sshHostKeyPolicyKey]),
+			ReadOnlyKnownHosts: readOnlyKnownHosts,
+		})
+	})
+	_ = registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		forcePathStyle, err := strconv.ParseBool(cfg[s3ForcePathStyleKey])
+		if err != nil {
+			return nil, fmt.Errorf("s3 force_path_style: %w", err)
+		}
+		return s3adapter.New(ctx, s3adapter.Options{
+			Endpoint:        cfg[s3EndpointKey],
+			Bucket:          cfg[s3BucketKey],
+			Prefix:          cfg[s3PrefixKey],
+			Region:          cfg[s3RegionKey],
+			ForcePathStyle:  forcePathStyle,
+			AccessKeyID:     cfg[s3AccessKeyIDKey],
+			SecretAccessKey: cfg[s3SecretAccessKey],
+		})
+	})
+	return &backendFactory{registry: registry, environment: environment}
 }
 
 func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) {
-	if source.Backend != config.BackendLocal {
-		return nil, fmt.Errorf("source backend %s is not implemented for execution", source.Backend)
-	}
-	return f.registry.Open(ctx, source.Backend, storage.OpenConfig{storagePathKey: source.Path})
+	return f.openBackend(ctx, backendOpenSpecFromSource(source))
 }
 
 func (f *backendFactory) openDestination(ctx context.Context, destination config.Destination) (storage.Backend, error) {
-	if destination.Backend != config.BackendLocal {
-		return nil, fmt.Errorf("backend %s is not implemented for execution", destination.Backend)
-	}
-	return f.registry.Open(ctx, destination.Backend, storage.OpenConfig{storagePathKey: destination.Path})
+	return f.openBackend(ctx, backendOpenSpecFromDestination(destination))
 }
 
 func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storage.Backend, error) {
 	return f.registry.Open(ctx, config.BackendLocal, storage.OpenConfig{storagePathKey: path})
 }
+
+func (f *backendFactory) resolveCredentials(creds config.Credentials) (config.ResolvedCredentials, error) {
+	return f.environment.ResolveCredentials(creds)
+}
+
+func (f *backendFactory) openBackend(ctx context.Context, spec backendOpenSpec) (storage.Backend, error) {
+	if !backendExecutable(spec.backend) {
+		if spec.role == "source" {
+			return nil, fmt.Errorf("source backend %s is not implemented for execution", spec.backend)
+		}
+		return nil, fmt.Errorf("backend %s is not implemented for execution", spec.backend)
+	}
+	openConfig, err := f.openConfig(spec)
+	if err != nil {
+		return nil, err
+	}
+	return f.registry.Open(ctx, spec.backend, openConfig)
+}
+
+func backendExecutable(name string) bool {
+	return name == config.BackendLocal || name == config.BackendSSH || name == config.BackendS3
+}
+
+func (f *backendFactory) openConfig(spec backendOpenSpec) (storage.OpenConfig, error) {
+	cfg := storage.OpenConfig{storagePathKey: spec.path}
+	switch spec.backend {
+	case config.BackendSSH:
+		cfg[sshHostKey] = spec.host
+		cfg[sshUserKey] = spec.user
+		cfg[sshPortKey] = strconv.Itoa(spec.port)
+		cfg[sshKeyFileKey] = spec.ssh.KeyFile
+		cfg[sshKnownHostsKey] = spec.ssh.KnownHosts
+		cfg[sshHostKeyPolicyKey] = string(spec.ssh.HostKeyPolicy)
+		cfg[sshReadOnlyHostsKey] = strconv.FormatBool(f.readOnlyKnownHosts)
+	case config.BackendS3:
+		if err := f.addS3Config(cfg, spec); err != nil {
+			return nil, err
+		}
+	}
+	return cfg, nil
+}
+
+func (f *backendFactory) addS3Config(cfg storage.OpenConfig, spec backendOpenSpec) error {
+	cfg[s3EndpointKey] = spec.endpoint
+	cfg[s3BucketKey] = spec.bucket
+	cfg[s3PrefixKey] = spec.prefix
+	cfg[s3RegionKey] = spec.region
+	cfg[s3ForcePathStyleKey] = strconv.FormatBool(config.ForcePathStyle(spec.forcePath))
+	if spec.credentials.AccessKeyIDEnv != "" || spec.credentials.SecretAccessKeyEnv != "" {
+		resolved, err := f.resolveCredentials(spec.credentials)
+		if err != nil {
+			return err
+		}
+		cfg[s3AccessKeyIDKey] = resolved.AccessKeyID
+		cfg[s3SecretAccessKey] = resolved.SecretAccessKey
+	}
+	return nil
+}
+
+func backendOpenSpecFromSource(source config.Backend) backendOpenSpec {
+	return backendOpenSpec{
+		role:        "source",
+		backend:     source.Backend,
+		path:        source.Path,
+		host:        source.Host,
+		user:        source.User,
+		port:        source.Port,
+		ssh:         source.SSH,
+		endpoint:    source.Endpoint,
+		bucket:      source.Bucket,
+		prefix:      source.Prefix,
+		region:      source.Region,
+		forcePath:   source.ForcePath,
+		credentials: source.Creds,
+	}
+}
+
+func backendOpenSpecFromDestination(destination config.Destination) backendOpenSpec {
+	return backendOpenSpec{
+		role:        "destination",
+		backend:     destination.Backend,
+		path:        destination.Path,
+		host:        destination.Host,
+		user:        destination.User,
+		port:        destination.Port,
+		ssh:         destination.SSH,
+		endpoint:    destination.Endpoint,
+		bucket:      destination.Bucket,
+		prefix:      destination.Prefix,
+		region:      destination.Region,
+		forcePath:   destination.ForcePath,
+		credentials: destination.Creds,
+	}
+}

internal/app/backends_test.go

@@ -1,11 +1,16 @@
 package app
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"strings"
 	"testing"
 
 	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
+	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
 )
 
 func TestBackendFactoryOpensLocalSource(t *testing.T) {
@@ -47,14 +52,125 @@ func TestBackendFactoryOpensDirectLocalPath(t *testing.T) {
 	}
 }
 
+func TestBackendFactoryOpensSSHSourceWithRegisteredOpener(t *testing.T) {
+	factory := &backendFactory{registry: storage.NewRegistry()}
+	var got storage.OpenConfig
+	if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		got = cfg
+		return fake.New(), nil
+	}); err != nil {
+		t.Fatalf("Register() error = %v", err)
+	}
+
+	backend, err := factory.openSource(context.Background(), config.Backend{
+		Backend: config.BackendSSH,
+		Host:    "source.example.com",
+		User:    "reports",
+		Port:    22,
+		Path:    "/reports",
+		SSH:     config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
+	})
+	if err != nil {
+		t.Fatalf("openSource() error = %v", err)
+	}
+	if backend == nil {
+		t.Fatal("openSource() backend = nil")
+	}
+	if got[sshHostKey] != "source.example.com" || got[storagePathKey] != "/reports" {
+		t.Fatalf("open config = %#v, want SSH source fields", got)
+	}
+}
+
+func TestBackendFactoryOpensSSHDestinationWithRegisteredOpener(t *testing.T) {
+	factory := &backendFactory{registry: storage.NewRegistry()}
+	var got storage.OpenConfig
+	if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		got = cfg
+		return fake.New(), nil
+	}); err != nil {
+		t.Fatalf("Register() error = %v", err)
+	}
+
+	backend, err := factory.openDestination(context.Background(), config.Destination{
+		Backend: config.BackendSSH,
+		Host:    "destination.example.com",
+		User:    "deploy",
+		Port:    2222,
+		Path:    "/archive",
+		SSH:     config.SSH{HostKeyPolicy: config.HostKeyPolicyStrict},
+	})
+	if err != nil {
+		t.Fatalf("openDestination() error = %v", err)
+	}
+	if backend == nil {
+		t.Fatal("openDestination() backend = nil")
+	}
+	if got[sshHostKey] != "destination.example.com" || got[sshPortKey] != "2222" || got[sshHostKeyPolicyKey] != "strict" {
+		t.Fatalf("open config = %#v, want SSH destination fields", got)
+	}
+}
+
+func TestBackendFactorySetsReadOnlyKnownHostsForDryRunSSH(t *testing.T) {
+	factory := &backendFactory{
+		registry:           storage.NewRegistry(),
+		readOnlyKnownHosts: true,
+	}
+	var got storage.OpenConfig
+	if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		got = cfg
+		return fake.New(), nil
+	}); err != nil {
+		t.Fatalf("Register() error = %v", err)
+	}
+
+	_, err := factory.openDestination(context.Background(), config.Destination{
+		Backend: config.BackendSSH,
+		Host:    "destination.example.com",
+		User:    "deploy",
+		Port:    22,
+		Path:    "/archive",
+		SSH:     config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
+	})
+	if err != nil {
+		t.Fatalf("openDestination() error = %v", err)
+	}
+	if got[sshReadOnlyHostsKey] != "true" {
+		t.Fatalf("open config %s = %q, want true", sshReadOnlyHostsKey, got[sshReadOnlyHostsKey])
+	}
+}
+
+func TestBackendFactoryUsesPersistentKnownHostsByDefault(t *testing.T) {
+	factory := &backendFactory{registry: storage.NewRegistry()}
+	var got storage.OpenConfig
+	if err := factory.registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		got = cfg
+		return fake.New(), nil
+	}); err != nil {
+		t.Fatalf("Register() error = %v", err)
+	}
+
+	_, err := factory.openDestination(context.Background(), config.Destination{
+		Backend: config.BackendSSH,
+		Host:    "destination.example.com",
+		User:    "deploy",
+		Port:    22,
+		Path:    "/archive",
+		SSH:     config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
+	})
+	if err != nil {
+		t.Fatalf("openDestination() error = %v", err)
+	}
+	if got[sshReadOnlyHostsKey] != "false" {
+		t.Fatalf("open config %s = %q, want false", sshReadOnlyHostsKey, got[sshReadOnlyHostsKey])
+	}
+}
+
 func TestBackendFactoryRejectsUnsupportedSource(t *testing.T) {
 	factory := newBackendFactory()
 	_, err := factory.openSource(context.Background(), config.Backend{
-		Backend: config.BackendSSH,
-		URI:     "ssh://reports@example.com:22",
-		Path:    "/reports",
+		Backend: "ftp",
 	})
-	if err == nil || !strings.Contains(err.Error(), "source backend ssh is not implemented for execution") {
+	if err == nil || !strings.Contains(err.Error(), "source backend ftp is not implemented for execution") {
 		t.Fatalf("openSource() error = %v, want not implemented", err)
 	}
 }
@@ -62,11 +178,461 @@ func TestBackendFactoryRejectsUnsupportedSource(t *testing.T) {
 func TestBackendFactoryRejectsUnsupportedDestination(t *testing.T) {
 	factory := newBackendFactory()
 	_, err := factory.openDestination(context.Background(), config.Destination{
-		Backend:  config.BackendS3,
-		Endpoint: "https://s3.example.com",
-		Bucket:   "reports",
+		Backend: "ftp",
 	})
-	if err == nil || !strings.Contains(err.Error(), "backend s3 is not implemented for execution") {
+	if err == nil || !strings.Contains(err.Error(), "backend ftp is not implemented for execution") {
 		t.Fatalf("openDestination() error = %v, want not implemented", err)
 	}
 }
+
+func TestBackendFactoryOpensS3DestinationWithRegisteredOpener(t *testing.T) {
+	factory := &backendFactory{
+		registry:    storage.NewRegistry(),
+		environment: config.NewEnvironment(nil, func(string) (string, bool) { return "", false }),
+	}
+	var got storage.OpenConfig
+	if err := factory.registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		got = cfg
+		return fake.New(), nil
+	}); err != nil {
+		t.Fatalf("Register() error = %v", err)
+	}
+	forcePathStyle := false
+	backend, err := factory.openDestination(context.Background(), config.Destination{
+		Backend:   config.BackendS3,
+		Endpoint:  "https://s3.example.com",
+		Bucket:    "reports",
+		Prefix:    "archive",
+		Region:    config.DefaultS3Region,
+		ForcePath: &forcePathStyle,
+	})
+	if err != nil {
+		t.Fatalf("openDestination() error = %v", err)
+	}
+	if backend == nil {
+		t.Fatal("openDestination() backend = nil")
+	}
+	assertOpenConfig(t, got, map[string]string{
+		s3EndpointKey:       "https://s3.example.com",
+		s3BucketKey:         "reports",
+		s3PrefixKey:         "archive",
+		s3RegionKey:         config.DefaultS3Region,
+		s3ForcePathStyleKey: "false",
+	})
+}
+
+func TestBackendFactoryResolvesS3CredentialsThroughSecretsAwareEnvironment(t *testing.T) {
+	factory := &backendFactory{
+		registry: storage.NewRegistry(),
+		environment: config.NewEnvironment(map[string]string{
+			"ACCESS_KEY_ID":     "secret-access",
+			"SECRET_ACCESS_KEY": "secret-secret",
+		}, func(string) (string, bool) { return "", false }),
+	}
+	var got storage.OpenConfig
+	if err := factory.registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+		got = cfg
+		return fake.New(), nil
+	}); err != nil {
+		t.Fatalf("Register() error = %v", err)
+	}
+	forcePathStyle := true
+	_, err := factory.openSource(context.Background(), config.Backend{
+		Backend:   config.BackendS3,
+		Endpoint:  "https://s3.example.com",
+		Bucket:    "reports",
+		Region:    config.DefaultS3Region,
+		ForcePath: &forcePathStyle,
+		Creds: config.Credentials{
+			AccessKeyIDEnv:     "ACCESS_KEY_ID",
+			SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
+		},
+	})
+	if err != nil {
+		t.Fatalf("openSource() error = %v", err)
+	}
+	assertOpenConfig(t, got, map[string]string{
+		s3AccessKeyIDKey:  "secret-access",
+		s3SecretAccessKey: "secret-secret",
+	})
+}
+
+func TestBackendFactoryResolvesCredentialsThroughEnvironment(t *testing.T) {
+	factory := newBackendFactoryWithEnvironment(config.NewEnvironment(map[string]string{
+		"ACCESS_KEY_ID":     "secret-access",
+		"SECRET_ACCESS_KEY": "secret-secret",
+	}, func(string) (string, bool) {
+		return "", false
+	}))
+
+	creds, err := factory.resolveCredentials(config.Credentials{
+		AccessKeyIDEnv:     "ACCESS_KEY_ID",
+		SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
+	})
+	if err != nil {
+		t.Fatalf("resolveCredentials() error = %v", err)
+	}
+	if creds.AccessKeyID != "secret-access" || creds.SecretAccessKey != "secret-secret" {
+		t.Fatalf("resolved credentials = %#v", creds)
+	}
+}
+
+func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) {
+	factory := &backendFactory{environment: config.NewEnvironment(nil, nil)}
+	cfg, err := factory.openConfig(backendOpenSpecFromSource(config.Backend{
+		Backend: config.BackendSSH,
+		Host:    "source.example.com",
+		User:    "reports",
+		Port:    2222,
+		Path:    "/reports",
+		SSH: config.SSH{
+			KeyFile:       "/home/reports/.ssh/id_ed25519",
+			KnownHosts:    "/home/reports/.ssh/known_hosts",
+			HostKeyPolicy: config.HostKeyPolicyStrict,
+		},
+	}))
+	if err != nil {
+		t.Fatalf("openConfig() error = %v", err)
+	}
+
+	assertOpenConfig(t, cfg, map[string]string{
+		storagePathKey:      "/reports",
+		sshHostKey:          "source.example.com",
+		sshUserKey:          "reports",
+		sshPortKey:          "2222",
+		sshKeyFileKey:       "/home/reports/.ssh/id_ed25519",
+		sshKnownHostsKey:    "/home/reports/.ssh/known_hosts",
+		sshHostKeyPolicyKey: "strict",
+	})
+}
+
+func TestBackendFactoryBuildsSSHDestinationOpenConfig(t *testing.T) {
+	factory := &backendFactory{environment: config.NewEnvironment(nil, nil)}
+	cfg, err := factory.openConfig(backendOpenSpecFromDestination(config.Destination{
+		Backend: config.BackendSSH,
+		Host:    "destination.example.com",
+		User:    "deploy",
+		Port:    22,
+		Path:    "/srv/archive",
+		SSH: config.SSH{
+			HostKeyPolicy: config.HostKeyPolicyAcceptNew,
+		},
+	}))
+	if err != nil {
+		t.Fatalf("openConfig() error = %v", err)
+	}
+
+	assertOpenConfig(t, cfg, map[string]string{
+		storagePathKey:      "/srv/archive",
+		sshHostKey:          "destination.example.com",
+		sshUserKey:          "deploy",
+		sshPortKey:          "22",
+		sshHostKeyPolicyKey: "accept-new",
+	})
+}
+
+func TestBackendFactoryBuildsEquivalentSourceAndDestinationOpenConfig(t *testing.T) {
+	forcePathStyle := false
+	tests := []struct {
+		name        string
+		source      config.Backend
+		destination config.Destination
+	}{
+		{
+			name:        "local",
+			source:      config.Backend{Backend: config.BackendLocal, Path: "/reports"},
+			destination: config.Destination{Backend: config.BackendLocal, Path: "/reports"},
+		},
+		{
+			name: "ssh",
+			source: config.Backend{
+				Backend: config.BackendSSH,
+				Host:    "reports.example.com",
+				User:    "reports",
+				Port:    2222,
+				Path:    "/reports",
+				SSH: config.SSH{
+					KeyFile:       "/home/reports/.ssh/id_ed25519",
+					KnownHosts:    "/home/reports/.ssh/known_hosts",
+					HostKeyPolicy: config.HostKeyPolicyStrict,
+				},
+			},
+			destination: config.Destination{
+				Backend: config.BackendSSH,
+				Host:    "reports.example.com",
+				User:    "reports",
+				Port:    2222,
+				Path:    "/reports",
+				SSH: config.SSH{
+					KeyFile:       "/home/reports/.ssh/id_ed25519",
+					KnownHosts:    "/home/reports/.ssh/known_hosts",
+					HostKeyPolicy: config.HostKeyPolicyStrict,
+				},
+			},
+		},
+		{
+			name: "s3",
+			source: config.Backend{
+				Backend:   config.BackendS3,
+				Endpoint:  "https://s3.example.com",
+				Bucket:    "reports",
+				Prefix:    "archive",
+				Region:    "us-west-2",
+				ForcePath: &forcePathStyle,
+			},
+			destination: config.Destination{
+				Backend:   config.BackendS3,
+				Endpoint:  "https://s3.example.com",
+				Bucket:    "reports",
+				Prefix:    "archive",
+				Region:    "us-west-2",
+				ForcePath: &forcePathStyle,
+			},
+		},
+		{
+			name: "s3 explicit credentials",
+			source: config.Backend{
+				Backend:  config.BackendS3,
+				Endpoint: "https://s3.example.com",
+				Bucket:   "reports",
+				Region:   config.DefaultS3Region,
+				Creds: config.Credentials{
+					AccessKeyIDEnv:     "ACCESS_KEY_ID",
+					SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
+				},
+			},
+			destination: config.Destination{
+				Backend:  config.BackendS3,
+				Endpoint: "https://s3.example.com",
+				Bucket:   "reports",
+				Region:   config.DefaultS3Region,
+				Creds: config.Credentials{
+					AccessKeyIDEnv:     "ACCESS_KEY_ID",
+					SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
+				},
+			},
+		},
+	}
+
+	factory := &backendFactory{
+		environment: config.NewEnvironment(map[string]string{
+			"ACCESS_KEY_ID":     "secret-access",
+			"SECRET_ACCESS_KEY": "secret-secret",
+		}, func(string) (string, bool) { return "", false }),
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sourceConfig, err := factory.openConfig(backendOpenSpecFromSource(tt.source))
+			if err != nil {
+				t.Fatalf("source openConfig() error = %v", err)
+			}
+			destinationConfig, err := factory.openConfig(backendOpenSpecFromDestination(tt.destination))
+			if err != nil {
+				t.Fatalf("destination openConfig() error = %v", err)
+			}
+			if !openConfigEqual(sourceConfig, destinationConfig) {
+				t.Fatalf("source open config = %#v, destination open config = %#v, want equivalent", sourceConfig, destinationConfig)
+			}
+		})
+	}
+}
+
+func TestBackendFactoryBuildsEquivalentDryRunSSHOpenConfig(t *testing.T) {
+	factory := &backendFactory{readOnlyKnownHosts: true}
+	sourceConfig, err := factory.openConfig(backendOpenSpecFromSource(config.Backend{
+		Backend: config.BackendSSH,
+		Host:    "reports.example.com",
+		Port:    22,
+		Path:    "/reports",
+		SSH:     config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
+	}))
+	if err != nil {
+		t.Fatalf("source openConfig() error = %v", err)
+	}
+	destinationConfig, err := factory.openConfig(backendOpenSpecFromDestination(config.Destination{
+		Backend: config.BackendSSH,
+		Host:    "reports.example.com",
+		Port:    22,
+		Path:    "/reports",
+		SSH:     config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
+	}))
+	if err != nil {
+		t.Fatalf("destination openConfig() error = %v", err)
+	}
+	if !openConfigEqual(sourceConfig, destinationConfig) {
+		t.Fatalf("source open config = %#v, destination open config = %#v, want equivalent", sourceConfig, destinationConfig)
+	}
+	if sourceConfig[sshReadOnlyHostsKey] != "true" {
+		t.Fatalf("open config %s = %q, want true", sshReadOnlyHostsKey, sourceConfig[sshReadOnlyHostsKey])
+	}
+}
+
+func TestConfiguredSourceValidationAndRunUseEquivalentSourceOpenConfig(t *testing.T) {
+	tests := []struct {
+		name       string
+		source     config.Backend
+		sourceKey  string
+		dest       config.Destination
+		destKey    string
+		wantFields map[string]string
+	}{
+		{
+			name: "s3",
+			source: config.Backend{
+				Backend:  config.BackendS3,
+				Endpoint: "https://s3.example.com",
+				Bucket:   "source-bucket",
+				Prefix:   "source-prefix",
+				Region:   config.DefaultS3Region,
+			},
+			sourceKey: "s3:source-bucket",
+			dest: config.Destination{
+				ID:       "archive",
+				Backend:  config.BackendS3,
+				Endpoint: "https://s3.example.com",
+				Bucket:   "destination-bucket",
+				Region:   config.DefaultS3Region,
+			},
+			destKey: "s3:destination-bucket",
+			wantFields: map[string]string{
+				s3EndpointKey:       "https://s3.example.com",
+				s3BucketKey:         "source-bucket",
+				s3PrefixKey:         "source-prefix",
+				s3RegionKey:         config.DefaultS3Region,
+				s3ForcePathStyleKey: "true",
+			},
+		},
+		{
+			name: "ssh",
+			source: config.Backend{
+				Backend: config.BackendSSH,
+				Host:    "ssh.example.com",
+				User:    "reports",
+				Port:    2222,
+				Path:    "/source",
+				SSH: config.SSH{
+					KeyFile:       "/home/reports/.ssh/id_ed25519",
+					KnownHosts:    "/home/reports/.ssh/known_hosts",
+					HostKeyPolicy: config.HostKeyPolicyStrict,
+				},
+			},
+			sourceKey: "ssh:/source",
+			dest: config.Destination{
+				ID:      "archive",
+				Backend: config.BackendSSH,
+				Host:    "ssh.example.com",
+				Port:    2222,
+				Path:    "/destination",
+				SSH:     config.SSH{HostKeyPolicy: config.HostKeyPolicyStrict},
+			},
+			destKey: "ssh:/destination",
+			wantFields: map[string]string{
+				storagePathKey:      "/source",
+				sshHostKey:          "ssh.example.com",
+				sshUserKey:          "reports",
+				sshPortKey:          "2222",
+				sshKeyFileKey:       "/home/reports/.ssh/id_ed25519",
+				sshKnownHostsKey:    "/home/reports/.ssh/known_hosts",
+				sshHostKeyPolicyKey: "strict",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sourceBackend := fake.New()
+			testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{ID: "reports.source"})
+			destinationBackend := fake.New()
+			var validateSourceConfig storage.OpenConfig
+			var runSourceConfig storage.OpenConfig
+			validateProvider := recordingBackendFactoryProvider(t, map[string]storage.Backend{
+				tt.sourceKey: sourceBackend,
+				tt.destKey:   destinationBackend,
+			}, func(cfg storage.OpenConfig) {
+				validateSourceConfig = cfg
+			})
+			runProvider := recordingBackendFactoryProvider(t, map[string]storage.Backend{
+				tt.sourceKey: sourceBackend,
+				tt.destKey:   destinationBackend,
+			}, func(cfg storage.OpenConfig) {
+				runSourceConfig = cfg
+			})
+			cfg := config.Config{Pipelines: []config.Pipeline{{
+				ID:           "reports",
+				Source:       tt.source,
+				Destinations: []config.Destination{tt.dest},
+			}}}
+			config.ApplyDefaults(&cfg)
+
+			var validateOutput bytes.Buffer
+			if err := validateConfigWithBackendFactory(context.Background(), cfg, ValidateOptions{
+				PipelineID: "reports",
+				Stdout:     &validateOutput,
+			}, validateProvider); err != nil {
+				t.Fatalf("validateConfigWithBackendFactory() error = %v", err)
+			}
+			if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{}, runProvider); err != nil {
+				t.Fatalf("runConfigWithBackendFactory() error = %v", err)
+			}
+			if !openConfigEqual(validateSourceConfig, runSourceConfig) {
+				t.Fatalf("validate source config = %#v, run source config = %#v, want equivalent", validateSourceConfig, runSourceConfig)
+			}
+			assertOpenConfig(t, runSourceConfig, tt.wantFields)
+		})
+	}
+}
+
+func assertOpenConfig(t *testing.T, got map[string]string, want map[string]string) {
+	t.Helper()
+	for key, wantValue := range want {
+		if gotValue := got[key]; gotValue != wantValue {
+			t.Fatalf("open config %s = %q, want %q", key, gotValue, wantValue)
+		}
+	}
+}
+
+func openConfigEqual(left, right storage.OpenConfig) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	for key, leftValue := range left {
+		if right[key] != leftValue {
+			return false
+		}
+	}
+	return true
+}
+
+func recordingBackendFactoryProvider(t *testing.T, remoteBackends map[string]storage.Backend, recordSource func(storage.OpenConfig)) backendFactoryProvider {
+	t.Helper()
+	return func(environment config.Environment) *backendFactory {
+		registry := storage.NewRegistry()
+		if err := registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+			if cfg[s3BucketKey] == "source-bucket" {
+				recordSource(cfg)
+			}
+			key := "s3:" + cfg[s3BucketKey]
+			backend := remoteBackends[key]
+			if backend == nil {
+				return nil, fmt.Errorf("missing fake backend for %s", key)
+			}
+			return backend, nil
+		}); err != nil {
+			t.Fatalf("register s3 backend: %v", err)
+		}
+		if err := registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
+			if cfg[storagePathKey] == "/source" {
+				recordSource(cfg)
+			}
+			key := "ssh:" + cfg[storagePathKey]
+			backend := remoteBackends[key]
+			if backend == nil {
+				return nil, fmt.Errorf("missing fake backend for %s", key)
+			}
+			return backend, nil
+		}); err != nil {
+			t.Fatalf("register ssh backend: %v", err)
+		}
+		return &backendFactory{registry: registry, environment: environment}
+	}
+}

internal/app/inspect.go

@@ -5,44 +5,100 @@ import (
 	"fmt"
 	"io"
 
-	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 )
 
 type InspectOptions struct {
-	Path   string
-	Stdout io.Writer
+	Path         string
+	ConfigPath   string
+	PipelineID   string
+	BundlePath   string
+	Stdout       io.Writer
+	OutputFormat OutputFormat
 }
 
 func Inspect(ctx context.Context, options InspectOptions) error {
-	if options.Path == "" {
-		return fmt.Errorf("inspect command requires a path")
-	}
-	backend, err := newBackendFactory().openLocalPath(ctx, options.Path)
-	if err != nil {
-		return err
-	}
-	bundles, err := bundle.Discover(ctx, backend, "")
-	if err != nil {
-		return err
-	}
-	return writeInspection(options.Stdout, bundles)
+	return inspectWithBackendFactory(ctx, options, newBackendFactoryWithEnvironment)
 }
 
-func writeInspection(w io.Writer, bundles []bundle.Bundle) error {
+func inspectWithBackendFactory(ctx context.Context, options InspectOptions, provider backendFactoryProvider) error {
+	if err := ValidateOutputFormat(options.OutputFormat); err != nil {
+		return err
+	}
+	selection, err := selectSourceBundles(ctx, sourceCommandOptions{
+		CommandName: "inspect",
+		Path:        options.Path,
+		ConfigPath:  options.ConfigPath,
+		PipelineID:  options.PipelineID,
+		BundlePath:  options.BundlePath,
+	}, provider)
+	if err != nil {
+		return err
+	}
+	return writeInspectResult(options, selection)
+}
+
+func inspectConfigWithBackendFactory(ctx context.Context, cfg config.Config, options InspectOptions, provider backendFactoryProvider) error {
+	if err := ValidateOutputFormat(options.OutputFormat); err != nil {
+		return err
+	}
+	selection, err := selectSourceBundlesFromConfig(ctx, cfg, sourceCommandOptions{
+		CommandName: "inspect",
+		PipelineID:  options.PipelineID,
+		BundlePath:  options.BundlePath,
+	}, provider)
+	if err != nil {
+		return err
+	}
+	return writeInspectResult(options, selection)
+}
+
+func writeInspectResult(options InspectOptions, selection sourceSelection) error {
+	if IsJSONOutput(options.OutputFormat) {
+		return WriteJSONEnvelope(options.Stdout, "inspect", true, selection.Warnings, inspectResultFromSelection(selection), nil)
+	}
+	if err := writeWarnings(options.Stdout, selection.Warnings); err != nil {
+		return err
+	}
+	return writeInspection(options.Stdout, selection)
+}
+
+type inspectResult struct {
+	PipelineID    string               `json:"pipeline_id,omitempty"`
+	SourceBackend string               `json:"source_backend,omitempty"`
+	BundleCount   int                  `json:"bundle_count"`
+	Bundles       []bundleDetailResult `json:"bundles"`
+}
+
+func inspectResultFromSelection(selection sourceSelection) inspectResult {
+	return inspectResult{
+		PipelineID:    selection.PipelineID,
+		SourceBackend: selection.SourceBackend,
+		BundleCount:   len(selection.Bundles),
+		Bundles:       bundleDetailsFromBundles(selection.Bundles),
+	}
+}
+
+func writeInspection(w io.Writer, selection sourceSelection) error {
 	if w == nil {
 		return nil
 	}
-	if _, err := fmt.Fprintf(w, "Bundles: %d\n", len(bundles)); err != nil {
+	if selection.ConfigMode {
+		if _, err := fmt.Fprintf(w, "Pipeline: %s\nSource: %s\n", selection.PipelineID, selection.SourceBackend); err != nil {
+			return err
+		}
+	}
+	if _, err := fmt.Fprintf(w, "Bundles: %d\n", len(selection.Bundles)); err != nil {
 		return err
 	}
-	for _, sourceBundle := range bundles {
+	for _, sourceBundle := range selection.Bundles {
 		if _, err := fmt.Fprintf(
 			w,
 			"- path=%s id=%s created=%s digest=%s files=%d\n",
 			storage.DisplayPath(sourceBundle.RootRelativePath),
 			sourceBundle.Manifest.ID,
-			sourceBundle.Manifest.Created.Format("2006-01-02T15:04:05Z07:00"),
+			formatManifestCreated(sourceBundle.Manifest.Created),
 			sourceBundle.Manifest.Digest,
 			len(sourceBundle.Manifest.Files),
 		); err != nil {

internal/app/inspect_test.go

@@ -3,9 +3,14 @@ package app
 import (
 	"bytes"
 	"context"
+	"encoding/json"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
 )
 
 func TestInspectPrintsBundleSummary(t *testing.T) {
@@ -32,6 +37,149 @@ func TestInspectPrintsBundleSummary(t *testing.T) {
 	}
 }
 
+func TestInspectConfiguredLocalSource(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "daily", testutil.BundleOptions{ID: "reports.daily"})
+	var stdout bytes.Buffer
+
+	err := Inspect(context.Background(), InspectOptions{
+		ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
+		PipelineID: "reports",
+		Stdout:     &stdout,
+	})
+
+	if err != nil {
+		t.Fatalf("Inspect() configured source error = %v", err)
+	}
+	output := stdout.String()
+	for _, want := range []string{
+		"Pipeline: reports",
+		"Source: local",
+		"Bundles: 1",
+		"path=daily",
+		"id=reports.daily",
+	} {
+		if !strings.Contains(output, want) {
+			t.Fatalf("Inspect() output = %q, want substring %q", output, want)
+		}
+	}
+}
+
+func TestInspectConfiguredSourceJSON(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{ID: "reports.json"})
+	var stdout bytes.Buffer
+
+	err := Inspect(context.Background(), InspectOptions{
+		ConfigPath:   testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
+		PipelineID:   "reports",
+		Stdout:       &stdout,
+		OutputFormat: OutputFormatJSON,
+	})
+
+	if err != nil {
+		t.Fatalf("Inspect() configured JSON error = %v", err)
+	}
+	result := decodeAppResult(t, stdout.String())
+	if result["pipeline_id"] != "reports" || result["source_backend"] != "local" || result["bundle_count"] != float64(1) {
+		t.Fatalf("result = %#v, want configured inspect metadata", result)
+	}
+}
+
+func TestInspectJSONPreservesCreatedOffsetAndFileMetadata(t *testing.T) {
+	sourceRoot := t.TempDir()
+	created := time.Date(2026, 6, 1, 6, 30, 0, 0, time.FixedZone("CDT", -5*60*60))
+	testutil.WriteSourceBundle(t, sourceRoot, "daily", testutil.BundleOptions{
+		ID:      "reports.offset",
+		Created: created,
+		Files: []testutil.SourceFile{
+			{Path: "report.md", Data: "# Report\n"},
+		},
+	})
+	var stdout bytes.Buffer
+
+	err := Inspect(context.Background(), InspectOptions{
+		Path:         sourceRoot,
+		Stdout:       &stdout,
+		OutputFormat: OutputFormatJSON,
+	})
+
+	if err != nil {
+		t.Fatalf("Inspect() error = %v", err)
+	}
+	result := decodeAppResult(t, stdout.String())
+	bundles, ok := result["bundles"].([]any)
+	if !ok || len(bundles) != 1 {
+		t.Fatalf("bundles = %#v, want one bundle", result["bundles"])
+	}
+	bundle, ok := bundles[0].(map[string]any)
+	if !ok {
+		t.Fatalf("bundle = %#v, want object", bundles[0])
+	}
+	if bundle["created"] != "2026-06-01T06:30:00-05:00" || bundle["file_count"] != float64(1) {
+		t.Fatalf("bundle = %#v, want offset timestamp and file count", bundle)
+	}
+	files, ok := bundle["files"].([]any)
+	if !ok || len(files) != 1 {
+		t.Fatalf("files = %#v, want one file", bundle["files"])
+	}
+	file, ok := files[0].(map[string]any)
+	if !ok || file["path"] != "report.md" || file["sha256"] == "" || file["size"] != float64(9) {
+		t.Fatalf("file = %#v, want projected file metadata", file)
+	}
+}
+
+func TestInspectConfiguredSourceJSONIncludesSecretConflictWarningWithoutValues(t *testing.T) {
+	name := "DISTRIBUTOR_TEST_INSPECT_SECRET"
+	t.Setenv(name, "process-value")
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	secretsRoot := t.TempDir()
+	if err := os.WriteFile(filepath.Join(secretsRoot, name), []byte("secret-value\n"), 0o600); err != nil {
+		t.Fatalf("write secret: %v", err)
+	}
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{ID: "reports.json"})
+	configPath := writeConfigFile(t, `
+secrets:
+  directory: `+secretsRoot+`
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: archive
+        backend: local
+        path: `+destinationRoot+`
+`)
+
+	var stdout bytes.Buffer
+	err := Inspect(context.Background(), InspectOptions{
+		ConfigPath:   configPath,
+		PipelineID:   "reports",
+		Stdout:       &stdout,
+		OutputFormat: OutputFormatJSON,
+	})
+	if err != nil {
+		t.Fatalf("Inspect() error = %v", err)
+	}
+	var envelope struct {
+		Warnings []OutputWarning `json:"warnings"`
+	}
+	if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil {
+		t.Fatalf("decode output: %v; output = %q", err, stdout.String())
+	}
+	if len(envelope.Warnings) != 1 || !strings.Contains(envelope.Warnings[0].Message, "secret "+name+" ignored") {
+		t.Fatalf("warnings = %#v, want secret conflict warning", envelope.Warnings)
+	}
+	output := stdout.String()
+	if strings.Contains(output, "process-value") || strings.Contains(output, "secret-value") {
+		t.Fatalf("stdout exposed secret values: %q", output)
+	}
+}
+
 func TestInspectRequiresPath(t *testing.T) {
 	err := Inspect(context.Background(), InspectOptions{})
 	if err == nil || !strings.Contains(err.Error(), "requires a path") {

internal/app/manifest.go

@@ -0,0 +1,114 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+	"time"
+
+	producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
+)
+
+type ManifestCreateOptions struct {
+	Root         string
+	ID           string
+	Created      string
+	Files        []string
+	Overwrite    bool
+	Stdout       io.Writer
+	OutputFormat OutputFormat
+}
+
+func ManifestCreate(ctx context.Context, options ManifestCreateOptions) error {
+	if err := ValidateOutputFormat(options.OutputFormat); err != nil {
+		return err
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if options.Root == "" {
+		return fmt.Errorf("manifest create command requires a bundle path")
+	}
+	if options.ID == "" {
+		return fmt.Errorf("manifest create command requires --id")
+	}
+
+	created, err := parseOptionalCreated(options.Created)
+	if err != nil {
+		return err
+	}
+	files := normalizeManifestFiles(options.Files)
+	buildOptions := producerbundle.BuildOptions{
+		Root:    options.Root,
+		ID:      options.ID,
+		Created: created,
+		Files:   files,
+		Scan:    len(files) == 0,
+	}
+	manifest, err := producerbundle.BuildManifest(buildOptions)
+	if err != nil {
+		return err
+	}
+	if err := producerbundle.WriteManifest(options.Root, manifest, producerbundle.WriteManifestOptions{Overwrite: options.Overwrite}); err != nil {
+		return err
+	}
+	loaded, err := producerbundle.LoadManifest(options.Root)
+	if err != nil {
+		return err
+	}
+	if err := producerbundle.ValidateBundle(options.Root, loaded); err != nil {
+		return err
+	}
+
+	result := manifestCreateResultFromManifest(options.Root, loaded)
+	if IsJSONOutput(options.OutputFormat) {
+		return WriteJSONEnvelope(options.Stdout, "manifest create", true, nil, result, nil)
+	}
+	if options.Stdout != nil {
+		_, err = fmt.Fprintf(options.Stdout, "created %s\nbundle: %s\nfiles: %d\ndigest: %s\n", producerbundle.ManifestName, result.ID, result.FileCount, result.Digest)
+	}
+	return err
+}
+
+func parseOptionalCreated(value string) (time.Time, error) {
+	if value == "" {
+		return time.Time{}, nil
+	}
+	created, err := time.Parse(time.RFC3339, value)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("created must be RFC3339: %w", err)
+	}
+	return created, nil
+}
+
+func normalizeManifestFiles(files []string) []string {
+	normalized := make([]string, 0, len(files))
+	for _, file := range files {
+		normalized = append(normalized, filepath.ToSlash(filepath.Clean(strings.ReplaceAll(file, "\\", string(filepath.Separator)))))
+	}
+	return normalized
+}
+
+type manifestCreateResult struct {
+	ManifestPath string               `json:"manifest_path"`
+	Root         string               `json:"root"`
+	ID           string               `json:"id"`
+	Created      string               `json:"created"`
+	Digest       string               `json:"digest"`
+	FileCount    int                  `json:"file_count"`
+	Files        []manifestFileResult `json:"files"`
+}
+
+func manifestCreateResultFromManifest(root string, manifest producerbundle.Manifest) manifestCreateResult {
+	return manifestCreateResult{
+		ManifestPath: filepath.ToSlash(filepath.Join(root, producerbundle.ManifestName)),
+		Root:         filepath.ToSlash(root),
+		ID:           manifest.ID,
+		Created:      formatManifestCreated(manifest.Created),
+		Digest:       manifest.Digest,
+		FileCount:    len(manifest.Files),
+		Files:        manifestFileResults(manifest.Files),
+	}
+}

internal/app/manifest_test.go

@@ -0,0 +1,42 @@
+package app
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestManifestCreateJSONPreservesCreatedOffsetAndFileMetadata(t *testing.T) {
+	root := t.TempDir()
+	if err := os.WriteFile(filepath.Join(root, "report.md"), []byte("# Report\n"), 0o600); err != nil {
+		t.Fatalf("write report: %v", err)
+	}
+	var stdout bytes.Buffer
+
+	err := ManifestCreate(context.Background(), ManifestCreateOptions{
+		Root:         root,
+		ID:           "reports.offset",
+		Created:      "2026-06-01T06:30:00-05:00",
+		Files:        []string{"report.md"},
+		Stdout:       &stdout,
+		OutputFormat: OutputFormatJSON,
+	})
+
+	if err != nil {
+		t.Fatalf("ManifestCreate() error = %v", err)
+	}
+	result := decodeAppResult(t, stdout.String())
+	if result["id"] != "reports.offset" || result["created"] != "2026-06-01T06:30:00-05:00" || result["file_count"] != float64(1) {
+		t.Fatalf("result = %#v, want manifest metadata", result)
+	}
+	files, ok := result["files"].([]any)
+	if !ok || len(files) != 1 {
+		t.Fatalf("files = %#v, want one file", result["files"])
+	}
+	file, ok := files[0].(map[string]any)
+	if !ok || file["path"] != "report.md" || file["sha256"] == "" || file["size"] != float64(9) {
+		t.Fatalf("file = %#v, want projected file metadata", file)
+	}
+}

internal/app/output.go

@@ -0,0 +1,76 @@
+package app
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+const outputSchemaVersion = 1
+
+type OutputFormat string
+
+const (
+	OutputFormatText OutputFormat = "text"
+	OutputFormatJSON OutputFormat = "json"
+)
+
+type OutputWarning struct {
+	Message string `json:"message"`
+}
+
+type OutputError struct {
+	PipelineID    string `json:"pipeline_id,omitempty"`
+	DestinationID string `json:"destination_id,omitempty"`
+	Backend       string `json:"backend,omitempty"`
+	BundlePath    string `json:"bundle_path,omitempty"`
+	Message       string `json:"message"`
+}
+
+type outputEnvelope struct {
+	SchemaVersion int             `json:"schema_version"`
+	Command       string          `json:"command"`
+	OK            bool            `json:"ok"`
+	Warnings      []OutputWarning `json:"warnings"`
+	Result        any             `json:"result"`
+	Errors        []OutputError   `json:"errors,omitempty"`
+}
+
+func NormalizeOutputFormat(format OutputFormat) OutputFormat {
+	if format == "" {
+		return OutputFormatText
+	}
+	return format
+}
+
+func ValidateOutputFormat(format OutputFormat) error {
+	switch NormalizeOutputFormat(format) {
+	case OutputFormatText, OutputFormatJSON:
+		return nil
+	default:
+		return fmt.Errorf("format must be text or json")
+	}
+}
+
+func IsJSONOutput(format OutputFormat) bool {
+	return NormalizeOutputFormat(format) == OutputFormatJSON
+}
+
+func WriteJSONEnvelope(w io.Writer, command string, ok bool, warnings []OutputWarning, result any, errors []OutputError) error {
+	if w == nil {
+		return nil
+	}
+	if warnings == nil {
+		warnings = []OutputWarning{}
+	}
+	envelope := outputEnvelope{
+		SchemaVersion: outputSchemaVersion,
+		Command:       command,
+		OK:            ok,
+		Warnings:      warnings,
+		Result:        result,
+		Errors:        errors,
+	}
+	encoder := json.NewEncoder(w)
+	return encoder.Encode(envelope)
+}

internal/app/output_projection.go

@@ -0,0 +1,83 @@
+package app
+
+import (
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+type bundleSummaryResult struct {
+	Path string `json:"path"`
+	ID   string `json:"id"`
+}
+
+type bundleDetailResult struct {
+	Path      string               `json:"path"`
+	ID        string               `json:"id"`
+	Created   string               `json:"created"`
+	Digest    string               `json:"digest"`
+	FileCount int                  `json:"file_count"`
+	TotalSize int64                `json:"total_size"`
+	Files     []manifestFileResult `json:"files"`
+}
+
+type manifestFileResult struct {
+	Path   string `json:"path"`
+	SHA256 string `json:"sha256"`
+	Size   int64  `json:"size"`
+}
+
+func bundleSummaryFromBundle(sourceBundle bundle.Bundle) bundleSummaryResult {
+	return bundleSummaryResult{
+		Path: storage.DisplayPath(sourceBundle.RootRelativePath),
+		ID:   sourceBundle.Manifest.ID,
+	}
+}
+
+func bundleSummariesFromBundles(sourceBundles []bundle.Bundle) []bundleSummaryResult {
+	results := make([]bundleSummaryResult, 0, len(sourceBundles))
+	for _, sourceBundle := range sourceBundles {
+		results = append(results, bundleSummaryFromBundle(sourceBundle))
+	}
+	return results
+}
+
+func bundleDetailFromBundle(sourceBundle bundle.Bundle) bundleDetailResult {
+	result := bundleDetailResult{
+		Path:      storage.DisplayPath(sourceBundle.RootRelativePath),
+		ID:        sourceBundle.Manifest.ID,
+		Created:   formatManifestCreated(sourceBundle.Manifest.Created),
+		Digest:    sourceBundle.Manifest.Digest,
+		FileCount: len(sourceBundle.Manifest.Files),
+		Files:     manifestFileResults(sourceBundle.Manifest.Files),
+	}
+	for _, file := range sourceBundle.Manifest.Files {
+		result.TotalSize += file.Size
+	}
+	return result
+}
+
+func bundleDetailsFromBundles(sourceBundles []bundle.Bundle) []bundleDetailResult {
+	results := make([]bundleDetailResult, 0, len(sourceBundles))
+	for _, sourceBundle := range sourceBundles {
+		results = append(results, bundleDetailFromBundle(sourceBundle))
+	}
+	return results
+}
+
+func manifestFileResults(files []bundle.ManifestFile) []manifestFileResult {
+	results := make([]manifestFileResult, 0, len(files))
+	for _, file := range files {
+		results = append(results, manifestFileResult{
+			Path:   file.Path,
+			SHA256: file.SHA256,
+			Size:   file.Size,
+		})
+	}
+	return results
+}
+
+func formatManifestCreated(created time.Time) string {
+	return created.Format(time.RFC3339)
+}

internal/app/run.go

@@ -2,272 +2,305 @@ package app
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
-	"strings"
 
 	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
 	"gitea.maximumdirect.net/eric/distributor/internal/config"
 	"gitea.maximumdirect.net/eric/distributor/internal/notify"
-	"gitea.maximumdirect.net/eric/distributor/internal/publish"
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 )
 
 type RunOptions struct {
+	ConfigPath   string
+	DryRun       bool
+	Force        bool
+	Stdout       io.Writer
+	OutputFormat OutputFormat
+	Notifier     notify.Notifier
+}
+
+type RunPipelineOptions struct {
 	ConfigPath string
+	PipelineID string
 	DryRun     bool
-	Stdout     io.Writer
+	Force      bool
+	Notifier   notify.Notifier
+}
+
+type RunPipelineWithLocalSourceOptions struct {
+	ConfigPath string
+	PipelineID string
+	SourceRoot string
+	DryRun     bool
+	Force      bool
 	Notifier   notify.Notifier
 }
 
 func Run(ctx context.Context, options RunOptions) error {
+	if err := ValidateOutputFormat(options.OutputFormat); err != nil {
+		return err
+	}
 	if err := ctx.Err(); err != nil {
 		return err
 	}
 
-	configPath := options.ConfigPath
-	if configPath == "" {
-		configPath = config.DefaultConfigPath
-	}
-	cfg, err := config.LoadFile(configPath)
+	setup, err := loadRuntimeSetup(options.ConfigPath)
 	if err != nil {
 		return err
 	}
-	return runConfig(ctx, cfg, options)
+	return runSetup(ctx, setup, options)
+}
+
+func RunPipeline(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
+	if err := ctx.Err(); err != nil {
+		return RunReport{}, err
+	}
+
+	setup, err := loadRuntimeSetup(options.ConfigPath)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return runPipelineSetup(ctx, setup, options)
+}
+
+func RunPipelineWithLocalSource(ctx context.Context, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+	if err := ctx.Err(); err != nil {
+		return RunReport{}, err
+	}
+	if options.SourceRoot == "" {
+		return RunReport{}, fmt.Errorf("source root is required")
+	}
+
+	setup, err := loadRuntimeSetup(options.ConfigPath)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return runPipelineSetupWithLocalSource(ctx, setup, options)
 }
 
 func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return err
+	}
+	return runSetupWithBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
+}
+
+type backendFactoryProvider func(config.Environment) *backendFactory
+
+func runPipelineConfig(ctx context.Context, cfg config.Config, options RunPipelineOptions) (RunReport, error) {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return runPipelineSetupWithBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
+}
+
+func runPipelineConfigWithLocalSource(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return runPipelineSetupWithLocalSourceAndBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
+}
+
+func runPipelineConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineOptions, provider backendFactoryProvider) (RunReport, error) {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return runPipelineSetupWithBackendFactory(ctx, setup, options, provider)
+}
+
+func runPipelineSetup(ctx context.Context, setup runtimeSetup, options RunPipelineOptions) (RunReport, error) {
+	return runPipelineSetupWithBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
+}
+
+func runPipelineSetupWithBackendFactory(ctx context.Context, setup runtimeSetup, options RunPipelineOptions, provider backendFactoryProvider) (RunReport, error) {
+	pipeline, ok := findPipeline(setup.Config, options.PipelineID)
+	if !ok {
+		return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
+	}
+	return buildRunReportWithSetup(ctx, setup.withPipelines([]config.Pipeline{pipeline}), RunOptions{
+		DryRun:   options.DryRun,
+		Force:    options.Force,
+		Notifier: options.Notifier,
+	}, provider, nil)
+}
+
+func runPipelineConfigWithLocalSourceAndBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions, provider backendFactoryProvider) (RunReport, error) {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return runPipelineSetupWithLocalSourceAndBackendFactory(ctx, setup, options, provider)
+}
+
+func runPipelineSetupWithLocalSource(ctx context.Context, setup runtimeSetup, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+	return runPipelineSetupWithLocalSourceAndBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
+}
+
+func runPipelineSetupWithLocalSourceAndBackendFactory(ctx context.Context, setup runtimeSetup, options RunPipelineWithLocalSourceOptions, provider backendFactoryProvider) (RunReport, error) {
+	pipeline, ok := findPipeline(setup.Config, options.PipelineID)
+	if !ok {
+		return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
+	}
+	return buildRunReportWithSetup(ctx, setup.withPipelines([]config.Pipeline{pipeline}), RunOptions{
+		DryRun:   options.DryRun,
+		Force:    options.Force,
+		Notifier: options.Notifier,
+	}, provider, &localSourceRoot{
+		pipelineID: options.PipelineID,
+		root:       options.SourceRoot,
+	})
+}
+
+func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) error {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return err
+	}
+	return runSetupWithBackendFactory(ctx, setup, options, provider)
+}
+
+func runSetup(ctx context.Context, setup runtimeSetup, options RunOptions) error {
+	return runSetupWithBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
+}
+
+func runSetupWithBackendFactory(ctx context.Context, setup runtimeSetup, options RunOptions, provider backendFactoryProvider) error {
+	report, err := buildRunReportWithSetup(ctx, setup, options, provider, nil)
+	if err != nil && !IsPartialResultError(err) {
+		return err
+	}
+	if outputErr := WriteRunReport(options.Stdout, options.OutputFormat, report); outputErr != nil {
+		return outputErr
+	}
+	return err
+}
+
+func buildRunReportWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) (RunReport, error) {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return buildRunReportWithSetup(ctx, setup, options, provider, nil)
+}
+
+type localSourceRoot struct {
+	pipelineID string
+	root       string
+}
+
+func buildRunReport(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider, sourceRoot *localSourceRoot) (RunReport, error) {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return buildRunReportWithSetup(ctx, setup, options, provider, sourceRoot)
+}
+
+func buildRunReportWithSetup(ctx context.Context, setup runtimeSetup, options RunOptions, provider backendFactoryProvider, sourceRoot *localSourceRoot) (RunReport, error) {
 	notifier := options.Notifier
 	if notifier == nil {
 		notifier = notify.Noop{}
 	}
 	summary := runSummary{dryRun: options.DryRun}
+	report := RunReport{
+		DryRun:    options.DryRun,
+		Pipelines: []RunPipelineSummary{},
+		Actions:   []RunActionRecord{},
+	}
 	var failures runFailures
-	backends := newBackendFactory()
+	recorder := runReportRecorder{
+		report:   &report,
+		summary:  &summary,
+		failures: &failures,
+	}
+	report.PreambleWarnings = append(report.PreambleWarnings, setup.Warnings...)
+	report.addWarnings(setup.Warnings)
+	backends := provider(setup.Environment)
+	backends.readOnlyKnownHosts = options.DryRun
 	transforms := newTransformRegistry()
-	if options.Stdout != nil {
-		if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
-			return err
-		}
-	}
-	for _, pipeline := range cfg.Pipelines {
-		sourceBackend, err := backends.openSource(ctx, pipeline.Source)
+	for _, pipeline := range setup.Config.Pipelines {
+		pipelineWarnings := sshWarnings(pipeline)
+		report.addWarnings(pipelineWarnings)
+		sourceBackend, bundles, sourceBackendName, err := openPipelineSource(ctx, backends, pipeline, sourceRoot)
 		if err != nil {
-			return fmt.Errorf("pipeline %s: %w", pipeline.ID, err)
+			return report, err
 		}
-		bundles, err := bundle.Discover(ctx, sourceBackend, "")
-		if err != nil {
-			return fmt.Errorf("pipeline %s discover source bundles: %w", pipeline.ID, err)
-		}
-		if options.Stdout != nil {
-			if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
-				return err
-			}
-		}
-		for _, sourceBundle := range bundles {
-			for _, destination := range pipeline.Destinations {
-				destinationBackend, err := backends.openDestination(ctx, destination)
-				if err != nil {
-					failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
-					summary.recordFailure()
-					if options.Stdout != nil {
-						writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, err)
-					}
-					continue
-				}
-				req := publish.Request{
-					PipelineID:            pipeline.ID,
-					DestinationID:         destination.ID,
-					SourceBundle:          sourceBundle,
-					SourceBackend:         sourceBackend,
-					DestinationBackend:    destinationBackend,
-					DestinationBundlePath: sourceBundle.RootRelativePath,
-					Publish:               *destination.Publish,
-					Transform:             destination.Transform,
-					Transformers:          transforms,
-					Transfer:              destination.Transfer,
-					DistributorVersion:    Version,
-				}
-				plan, err := publish.Build(ctx, req)
-				if err != nil && plan.DestinationID == "" {
-					plan = publish.Plan{DestinationID: destination.ID, BundlePath: sourceBundle.RootRelativePath}
-				}
-				if options.Stdout != nil {
-					writePlanLine(options.Stdout, plan, err)
-				}
-				if err != nil {
-					failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
-					summary.recordFailure()
-					continue
-				}
-				summary.recordPlan(plan.Action)
-				if !options.DryRun {
-					if err := publish.Execute(ctx, req, plan); err != nil {
-						failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
-						summary.recordFailure()
-						continue
-					}
-					if shouldNotify(plan.Action) {
-						if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
-							failures.add(pipeline.ID, destination.ID, storage.DisplayPath(sourceBundle.RootRelativePath), err)
-							summary.recordFailure()
-							continue
-						}
-					}
-				}
-			}
-		}
-	}
-	if options.Stdout != nil {
-		if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil {
-			return err
-		}
-	}
-	if len(failures.items) > 0 {
-		return failures
-	}
-	return nil
-}
-
-func writePlanLine(w io.Writer, plan publish.Plan, planErr error) {
-	if w == nil {
-		return
-	}
-	if planErr != nil {
-		destinationID := plan.DestinationID
-		if destinationID == "" {
-			destinationID = "unknown"
-		}
-		fmt.Fprintf(w, "  - bundle=%s destination=%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, planErr.Error())
-		return
-	}
-	fmt.Fprintf(w, "  - bundle=%s destination=%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, plan.Action, outputSummary(plan.Outputs), plan.Reason)
-}
-
-func writeErrorLine(w io.Writer, bundlePath, destinationID string, err error) {
-	if w == nil {
-		return
-	}
-	fmt.Fprintf(w, "  - bundle=%s destination=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, err.Error())
-}
-
-func outputSummary(outputs []publish.Output) string {
-	if len(outputs) == 0 {
-		return "none"
-	}
-	paths := make([]string, 0, len(outputs))
-	for _, output := range outputs {
-		paths = append(paths, output.DestinationPath)
-	}
-	return strings.Join(paths, ",")
-}
-
-func destinationSummary(destinations []config.Destination) string {
-	if len(destinations) == 0 {
-		return "none"
-	}
-	ids := make([]string, 0, len(destinations))
-	for _, destination := range destinations {
-		ids = append(ids, destination.ID)
-	}
-	return strings.Join(ids, ",")
-}
-
-func shouldNotify(action publish.Action) bool {
-	return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder
-}
-
-func notifyEvent(plan publish.Plan) notify.Event {
-	outputs := make([]notify.Output, 0, len(plan.Outputs))
-	for _, output := range plan.Outputs {
-		outputs = append(outputs, notify.Output{
-			Path:       output.DestinationPath,
-			Kind:       output.Kind,
-			SourcePath: output.SourcePath,
-			Transform:  output.Transform,
-			SHA256:     output.SHA256,
-			Size:       output.Size,
+		report.Pipelines = append(report.Pipelines, RunPipelineSummary{
+			ID:            pipeline.ID,
+			SourceBackend: sourceBackendName,
+			BundleCount:   len(bundles),
+			Destinations:  destinationIDs(pipeline.Destinations),
+			Warnings:      pipelineWarnings,
 		})
+		pipelineIndex := len(report.Pipelines) - 1
+		for _, destination := range pipeline.Destinations {
+			processDestination(ctx, runDestinationRequest{
+				options:       options,
+				notifier:      notifier,
+				backends:      backends,
+				transforms:    transforms,
+				pipeline:      pipeline,
+				pipelineIndex: pipelineIndex,
+				sourceBackend: sourceBackend,
+				bundles:       bundles,
+				destination:   destination,
+				recorder:      &recorder,
+			})
+		}
+		closeBackend(sourceBackend)
 	}
-	return notify.Event{
-		PipelineID:    plan.PipelineID,
-		DestinationID: plan.DestinationID,
-		BundleID:      plan.BundleID,
-		BundlePath:    plan.BundlePath,
-		Action:        string(plan.Action),
-		Outputs:       outputs,
+	report.Summary = summary.Result()
+	report.OutputErrors = failures.outputErrors()
+	if len(failures.items) > 0 {
+		return report, failures
 	}
+	return report, nil
 }
 
-type runSummary struct {
-	dryRun       bool
-	planned      int
-	publishNew   int
-	replaceOlder int
-	skipped      int
-	failures     int
+type runReportRecorder struct {
+	report   *RunReport
+	summary  *runSummary
+	failures *runFailures
 }
 
-func (s *runSummary) recordPlan(action publish.Action) {
-	s.planned++
-	switch action {
-	case publish.ActionPublishNew:
-		s.publishNew++
-	case publish.ActionReplaceOlder:
-		s.replaceOlder++
-	case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
-		s.skipped++
+func openPipelineSource(ctx context.Context, backends *backendFactory, pipeline config.Pipeline, sourceRoot *localSourceRoot) (storage.Backend, []bundle.Bundle, string, error) {
+	if sourceRoot != nil && sourceRoot.pipelineID == pipeline.ID {
+		sourceBackend, err := backends.openLocalPath(ctx, sourceRoot.root)
+		if err != nil {
+			return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, config.BackendLocal, err)
+		}
+		sourceBundle, err := bundle.Validate(ctx, sourceBackend, "")
+		if err != nil {
+			closeBackend(sourceBackend)
+			return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s validate source bundle: %w", pipeline.ID, config.BackendLocal, err)
+		}
+		return sourceBackend, []bundle.Bundle{sourceBundle}, config.BackendLocal, nil
 	}
-}
 
-func (s *runSummary) recordFailure() {
-	s.failures++
-}
-
-func (s runSummary) Line() string {
-	status := "ok"
-	if s.failures > 0 {
-		status = "failed"
+	sourceBackend, err := backends.openSource(ctx, pipeline.Source)
+	if err != nil {
+		return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
 	}
-	return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d skipped=%d failed=%d dry_run=%t", status, s.planned, s.publishNew, s.replaceOlder, s.skipped, s.failures, s.dryRun)
-}
-
-type runFailure struct {
-	pipelineID    string
-	destinationID string
-	bundlePath    string
-	err           error
-}
-
-type runFailures struct {
-	items []runFailure
-}
-
-func (f *runFailures) add(pipelineID, destinationID, bundlePath string, err error) {
-	f.items = append(f.items, runFailure{
-		pipelineID:    pipelineID,
-		destinationID: destinationID,
-		bundlePath:    bundlePath,
-		err:           err,
-	})
-}
-
-func (f runFailures) Error() string {
-	if len(f.items) == 0 {
-		return ""
+	bundles, err := bundle.Discover(ctx, sourceBackend, "")
+	if err != nil {
+		closeBackend(sourceBackend)
+		return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
 	}
-	parts := make([]string, 0, len(f.items))
-	for _, item := range f.items {
-		parts = append(parts, fmt.Sprintf("pipeline %s destination %s bundle %s: %v", item.pipelineID, item.destinationID, item.bundlePath, item.err))
-	}
-	return "run failed: " + strings.Join(parts, "; ")
+	return sourceBackend, bundles, pipeline.Source.Backend, nil
 }
 
-func (f runFailures) Unwrap() error {
-	errs := make([]error, 0, len(f.items))
-	for _, item := range f.items {
-		errs = append(errs, item.err)
-	}
-	return errors.Join(errs...)
+type closeableBackend interface {
+	Close() error
+}
+
+func closeBackend(backend storage.Backend) {
+	closeable, ok := backend.(closeableBackend)
+	if !ok {
+		return
+	}
+	_ = closeable.Close()
 }

internal/app/run_destination.go

@@ -0,0 +1,163 @@
+package app
+
+import (
+	"context"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/notify"
+	"gitea.maximumdirect.net/eric/distributor/internal/publish"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+type runDestinationRequest struct {
+	options       RunOptions
+	notifier      notify.Notifier
+	backends      *backendFactory
+	transforms    publish.TransformerResolver
+	pipeline      config.Pipeline
+	pipelineIndex int
+	sourceBackend storage.Backend
+	bundles       []bundle.Bundle
+	destination   config.Destination
+	recorder      *runReportRecorder
+}
+
+func processDestination(ctx context.Context, request runDestinationRequest) {
+	selections := selectDestinationBundles(request.destination, request.bundles)
+	if isFixedPathDestination(request.destination) {
+		request.recorder.summary.recordFixedPath()
+		if request.options.DryRun {
+			warning := fixedPathSelectionWarning(request.pipeline.ID, request.destination.ID, selections, len(request.bundles))
+			request.recorder.addPipelineWarning(request.pipelineIndex, warning)
+		}
+	}
+	if len(selections) == 0 {
+		return
+	}
+	destinationBackend, err := request.backends.openDestination(ctx, request.destination)
+	if err != nil {
+		for _, selection := range selections {
+			sourceBundle := selection.SourceBundle
+			request.recorder.recordDestinationFailure(request.pipelineIndex, runFailure{
+				pipelineID:    request.pipeline.ID,
+				destinationID: request.destination.ID,
+				backend:       request.destination.Backend,
+				bundlePath:    sourceBundle.RootRelativePath,
+				err:           err,
+			}, errorAction(request.pipeline.ID, request.destination.ID, request.destination.Backend, sourceBundle.RootRelativePath, err), true)
+		}
+		return
+	}
+	defer closeBackend(destinationBackend)
+
+	for _, selection := range selections {
+		processDestinationSelection(ctx, request, destinationBackend, selection)
+	}
+}
+
+func processDestinationSelection(ctx context.Context, request runDestinationRequest, destinationBackend storage.Backend, selection destinationBundleSelection) {
+	sourceBundle := selection.SourceBundle
+	publishRequest := publish.Request{
+		PipelineID:            request.pipeline.ID,
+		DestinationID:         request.destination.ID,
+		SourceBundle:          sourceBundle,
+		SourceBackend:         request.sourceBackend,
+		DestinationBackend:    destinationBackend,
+		DestinationBundlePath: selection.DestinationBundlePath,
+		PathMapping:           request.destination.PathMap.Mode,
+		Publish:               *request.destination.Publish,
+		Transform:             request.destination.Transform,
+		Links:                 request.destination.Links,
+		Transformers:          request.transforms,
+		Transfer:              request.destination.Transfer,
+		DistributorVersion:    Version,
+		Force:                 request.options.Force,
+	}
+	plan, err := publish.Build(ctx, publishRequest)
+	if err != nil {
+		plan = completePlanIdentity(plan, request.pipeline, request.destination, selection)
+	}
+	if isFixedPathDestination(request.destination) {
+		plan.PathMapping = config.PathMappingFixed
+		if request.options.DryRun && isDestructiveFixedPathAction(plan.Action) {
+			warning := fixedPathReplacementWarning(plan)
+			request.recorder.addPipelineWarning(request.pipelineIndex, warning)
+		}
+	}
+	action := runActionFromPlan(request.destination.Backend, plan, err)
+	if err != nil {
+		request.recorder.recordDestinationFailure(request.pipelineIndex, runFailure{
+			pipelineID:    request.pipeline.ID,
+			destinationID: request.destination.ID,
+			backend:       request.destination.Backend,
+			bundlePath:    sourceBundle.RootRelativePath,
+			err:           err,
+		}, action, true)
+		return
+	}
+	request.recorder.addPipelineAction(request.pipelineIndex, action)
+	request.recorder.summary.recordPlan(plan.Action)
+	if request.options.DryRun {
+		return
+	}
+	if err := publish.Execute(ctx, publishRequest, plan); err != nil {
+		request.recorder.recordDestinationFailure(request.pipelineIndex, runFailure{
+			pipelineID:    request.pipeline.ID,
+			destinationID: request.destination.ID,
+			backend:       request.destination.Backend,
+			bundlePath:    sourceBundle.RootRelativePath,
+			err:           err,
+		}, RunActionRecord{}, false)
+		return
+	}
+	if shouldNotify(plan.Action) {
+		if err := request.notifier.Notify(ctx, notifyEvent(plan)); err != nil {
+			request.recorder.recordDestinationFailure(request.pipelineIndex, runFailure{
+				pipelineID:    request.pipeline.ID,
+				destinationID: request.destination.ID,
+				backend:       request.destination.Backend,
+				bundlePath:    sourceBundle.RootRelativePath,
+				err:           err,
+			}, RunActionRecord{}, false)
+			return
+		}
+	}
+}
+
+func (recorder *runReportRecorder) addPipelineWarning(pipelineIndex int, warning OutputWarning) {
+	recorder.report.addWarning(warning)
+	recorder.report.Pipelines[pipelineIndex].events = append(recorder.report.Pipelines[pipelineIndex].events, warningEvent(warning))
+}
+
+func (recorder *runReportRecorder) addPipelineAction(pipelineIndex int, action RunActionRecord) {
+	recorder.report.Actions = append(recorder.report.Actions, action)
+	recorder.report.Pipelines[pipelineIndex].events = append(recorder.report.Pipelines[pipelineIndex].events, actionEvent(len(recorder.report.Actions)-1))
+}
+
+func (recorder *runReportRecorder) recordDestinationFailure(pipelineIndex int, failure runFailure, action RunActionRecord, includeAction bool) {
+	recorder.failures.add(failure.pipelineID, failure.destinationID, failure.backend, storage.DisplayPath(failure.bundlePath), failure.err)
+	recorder.summary.recordFailure()
+	if includeAction {
+		recorder.addPipelineAction(pipelineIndex, action)
+	}
+}
+
+func completePlanIdentity(plan publish.Plan, pipeline config.Pipeline, destination config.Destination, selection destinationBundleSelection) publish.Plan {
+	if plan.PipelineID == "" {
+		plan.PipelineID = pipeline.ID
+	}
+	if plan.DestinationID == "" {
+		plan.DestinationID = destination.ID
+	}
+	if plan.BundleID == "" {
+		plan.BundleID = selection.SourceBundle.Manifest.ID
+	}
+	if plan.BundlePath == "" {
+		plan.BundlePath = selection.SourceBundle.RootRelativePath
+	}
+	if plan.DestinationBundlePath == "" {
+		plan.DestinationBundlePath = selection.DestinationBundlePath
+	}
+	return plan
+}

internal/app/run_failures.go

@@ -0,0 +1,70 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+type runFailure struct {
+	pipelineID    string
+	destinationID string
+	backend       string
+	bundlePath    string
+	err           error
+}
+
+type runFailures struct {
+	items []runFailure
+}
+
+func (f *runFailures) add(pipelineID, destinationID, backend, bundlePath string, err error) {
+	f.items = append(f.items, runFailure{
+		pipelineID:    pipelineID,
+		destinationID: destinationID,
+		backend:       backend,
+		bundlePath:    bundlePath,
+		err:           err,
+	})
+}
+
+func (f runFailures) Error() string {
+	if len(f.items) == 0 {
+		return ""
+	}
+	parts := make([]string, 0, len(f.items))
+	for _, item := range f.items {
+		parts = append(parts, fmt.Sprintf("pipeline %s destination %s backend %s bundle %s: %v", item.pipelineID, item.destinationID, item.backend, item.bundlePath, item.err))
+	}
+	return "run failed: " + strings.Join(parts, "; ")
+}
+
+func (f runFailures) outputErrors() []OutputError {
+	if len(f.items) == 0 {
+		return nil
+	}
+	errors := make([]OutputError, 0, len(f.items))
+	for _, item := range f.items {
+		errors = append(errors, OutputError{
+			PipelineID:    item.pipelineID,
+			DestinationID: item.destinationID,
+			Backend:       item.backend,
+			BundlePath:    item.bundlePath,
+			Message:       item.err.Error(),
+		})
+	}
+	return errors
+}
+
+func IsPartialResultError(err error) bool {
+	var failures runFailures
+	return errors.As(err, &failures)
+}
+
+func (f runFailures) Unwrap() error {
+	errs := make([]error, 0, len(f.items))
+	for _, item := range f.items {
+		errs = append(errs, item.err)
+	}
+	return errors.Join(errs...)
+}

internal/app/run_notify.go

@@ -0,0 +1,33 @@
+package app
+
+import (
+	"gitea.maximumdirect.net/eric/distributor/internal/notify"
+	"gitea.maximumdirect.net/eric/distributor/internal/publish"
+)
+
+func shouldNotify(action publish.Action) bool {
+	return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
+}
+
+func notifyEvent(plan publish.Plan) notify.Event {
+	outputs := make([]notify.Output, 0, len(plan.Outputs))
+	for _, output := range plan.Outputs {
+		stateOutput := output.StateOutputFile()
+		outputs = append(outputs, notify.Output{
+			Path:       stateOutput.Path,
+			Kind:       stateOutput.Kind,
+			SourcePath: stateOutput.SourcePath,
+			Transform:  stateOutput.Transform,
+			SHA256:     stateOutput.SHA256,
+			Size:       stateOutput.Size,
+		})
+	}
+	return notify.Event{
+		PipelineID:    plan.PipelineID,
+		DestinationID: plan.DestinationID,
+		BundleID:      plan.BundleID,
+		BundlePath:    plan.BundlePath,
+		Action:        string(plan.Action),
+		Outputs:       outputs,
+	}
+}

internal/app/run_output.go

@@ -0,0 +1,218 @@
+package app
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/publish"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+func WriteRunReport(w io.Writer, format OutputFormat, report RunReport) error {
+	if IsJSONOutput(format) {
+		return WriteJSONEnvelope(w, "run", len(report.OutputErrors) == 0, report.Warnings, report, report.OutputErrors)
+	}
+	return writeRunReportText(w, report)
+}
+
+func writeRunReportText(w io.Writer, report RunReport) error {
+	if w == nil {
+		return nil
+	}
+	if err := writeWarnings(w, report.PreambleWarnings); err != nil {
+		return err
+	}
+	if _, err := fmt.Fprintf(w, "Configured pipelines: %d\n", len(report.Pipelines)); err != nil {
+		return err
+	}
+	for _, pipeline := range report.Pipelines {
+		if err := writeWarnings(w, pipeline.Warnings); err != nil {
+			return err
+		}
+		if _, err := fmt.Fprintf(w, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.SourceBackend, pipeline.BundleCount, destinationIDSummary(pipeline.Destinations)); err != nil {
+			return err
+		}
+		for _, event := range pipeline.events {
+			if event.warning != nil {
+				if err := writeWarnings(w, []OutputWarning{*event.warning}); err != nil {
+					return err
+				}
+				continue
+			}
+			if event.actionIndex < 0 || event.actionIndex >= len(report.Actions) {
+				continue
+			}
+			writeRunActionLine(w, report.Actions[event.actionIndex])
+		}
+	}
+	_, err := fmt.Fprintln(w, report.Summary.Line())
+	return err
+}
+
+func writeRunActionLine(w io.Writer, action RunActionRecord) {
+	if action.Action == "error" {
+		destinationID := action.DestinationID
+		if destinationID == "" {
+			destinationID = "unknown"
+		}
+		fmt.Fprintf(w, "  - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", action.BundlePath, destinationID, action.Backend, pathMappingRecordSummary(action), action.Reason)
+		return
+	}
+	fmt.Fprintf(w, "  - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, outputRecordSummary(action.Outputs), action.Reason)
+}
+
+func pathMappingRecordSummary(action RunActionRecord) string {
+	if action.PathMapping != config.PathMappingFixed {
+		return ""
+	}
+	return fmt.Sprintf(" path_mapping=fixed target=%s", action.DestinationPath)
+}
+
+func outputRecordSummary(outputs []RunOutputRecord) string {
+	if len(outputs) == 0 {
+		return "none"
+	}
+	paths := make([]string, 0, len(outputs))
+	for _, output := range outputs {
+		paths = append(paths, output.Path)
+	}
+	return strings.Join(paths, ",")
+}
+
+func destinationIDSummary(ids []string) string {
+	if len(ids) == 0 {
+		return "none"
+	}
+	return strings.Join(ids, ",")
+}
+
+type RunReport struct {
+	DryRun           bool                 `json:"dry_run"`
+	Pipelines        []RunPipelineSummary `json:"pipelines"`
+	Actions          []RunActionRecord    `json:"actions"`
+	Summary          RunSummaryCounters   `json:"summary"`
+	Warnings         []OutputWarning      `json:"-"`
+	OutputErrors     []OutputError        `json:"-"`
+	PreambleWarnings []OutputWarning      `json:"-"`
+}
+
+func (r *RunReport) addWarning(warning OutputWarning) {
+	r.Warnings = append(r.Warnings, warning)
+}
+
+func (r *RunReport) addWarnings(warnings []OutputWarning) {
+	r.Warnings = append(r.Warnings, warnings...)
+}
+
+type RunPipelineSummary struct {
+	ID            string          `json:"id"`
+	SourceBackend string          `json:"source_backend"`
+	BundleCount   int             `json:"bundle_count"`
+	Destinations  []string        `json:"destinations"`
+	Warnings      []OutputWarning `json:"-"`
+	events        []runPipelineEvent
+}
+
+type runPipelineEvent struct {
+	warning     *OutputWarning
+	actionIndex int
+}
+
+func warningEvent(warning OutputWarning) runPipelineEvent {
+	return runPipelineEvent{warning: &warning, actionIndex: -1}
+}
+
+func actionEvent(actionIndex int) runPipelineEvent {
+	return runPipelineEvent{actionIndex: actionIndex}
+}
+
+type RunActionRecord struct {
+	PipelineID      string            `json:"pipeline_id,omitempty"`
+	DestinationID   string            `json:"destination_id"`
+	Backend         string            `json:"backend"`
+	BundleID        string            `json:"bundle_id,omitempty"`
+	BundlePath      string            `json:"bundle_path"`
+	DestinationPath string            `json:"destination_path"`
+	PathMapping     string            `json:"path_mapping,omitempty"`
+	Action          string            `json:"action"`
+	PrimaryURL      string            `json:"primary_url,omitempty"`
+	Reason          string            `json:"reason,omitempty"`
+	Outputs         []RunOutputRecord `json:"outputs"`
+}
+
+type RunOutputRecord struct {
+	Path       string `json:"path"`
+	Kind       string `json:"kind"`
+	SourcePath string `json:"source_path,omitempty"`
+	Transform  string `json:"transform,omitempty"`
+	URL        string `json:"url,omitempty"`
+	SHA256     string `json:"sha256"`
+	Size       int64  `json:"size"`
+}
+
+func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActionRecord {
+	if planErr != nil {
+		destinationID := plan.DestinationID
+		if destinationID == "" {
+			destinationID = "unknown"
+		}
+		return RunActionRecord{
+			PipelineID:      plan.PipelineID,
+			DestinationID:   destinationID,
+			Backend:         backend,
+			BundleID:        plan.BundleID,
+			BundlePath:      storage.DisplayPath(plan.BundlePath),
+			DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
+			PathMapping:     plan.PathMapping,
+			Action:          "error",
+			PrimaryURL:      plan.PrimaryURL,
+			Reason:          planErr.Error(),
+			Outputs:         []RunOutputRecord{},
+		}
+	}
+	return RunActionRecord{
+		PipelineID:      plan.PipelineID,
+		DestinationID:   plan.DestinationID,
+		Backend:         backend,
+		BundleID:        plan.BundleID,
+		BundlePath:      storage.DisplayPath(plan.BundlePath),
+		DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
+		PathMapping:     plan.PathMapping,
+		Action:          string(plan.Action),
+		PrimaryURL:      plan.PrimaryURL,
+		Reason:          plan.Reason,
+		Outputs:         runOutputsFromPlan(plan.Outputs),
+	}
+}
+
+func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) RunActionRecord {
+	return RunActionRecord{
+		PipelineID:      pipelineID,
+		DestinationID:   destinationID,
+		Backend:         backend,
+		BundlePath:      storage.DisplayPath(bundlePath),
+		DestinationPath: storage.DisplayPath(bundlePath),
+		Action:          "error",
+		Reason:          err.Error(),
+		Outputs:         []RunOutputRecord{},
+	}
+}
+
+func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
+	results := make([]RunOutputRecord, 0, len(outputs))
+	for _, output := range outputs {
+		stateOutput := output.StateOutputFile()
+		results = append(results, RunOutputRecord{
+			Path:       stateOutput.Path,
+			Kind:       stateOutput.Kind,
+			SourcePath: stateOutput.SourcePath,
+			Transform:  stateOutput.Transform,
+			URL:        stateOutput.URL,
+			SHA256:     stateOutput.SHA256,
+			Size:       stateOutput.Size,
+		})
+	}
+	return results
+}

internal/app/run_selection.go

@@ -0,0 +1,79 @@
+package app
+
+import (
+	"fmt"
+	"sort"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/publish"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+type destinationBundleSelection struct {
+	SourceBundle          bundle.Bundle
+	DestinationBundlePath string
+}
+
+func selectDestinationBundles(destination config.Destination, bundles []bundle.Bundle) []destinationBundleSelection {
+	if !isFixedPathDestination(destination) {
+		selections := make([]destinationBundleSelection, 0, len(bundles))
+		for _, sourceBundle := range bundles {
+			selections = append(selections, destinationBundleSelection{
+				SourceBundle:          sourceBundle,
+				DestinationBundlePath: sourceBundle.RootRelativePath,
+			})
+		}
+		return selections
+	}
+	if len(bundles) == 0 {
+		return nil
+	}
+	sourceBundle := newestBundle(bundles)
+	return []destinationBundleSelection{{
+		SourceBundle:          sourceBundle,
+		DestinationBundlePath: "",
+	}}
+}
+
+func newestBundle(bundles []bundle.Bundle) bundle.Bundle {
+	if len(bundles) == 0 {
+		return bundle.Bundle{}
+	}
+	sorted := append([]bundle.Bundle(nil), bundles...)
+	sort.Slice(sorted, func(i, j int) bool {
+		if sorted[i].Manifest.Created.Equal(sorted[j].Manifest.Created) {
+			return sorted[i].RootRelativePath < sorted[j].RootRelativePath
+		}
+		return sorted[i].Manifest.Created.After(sorted[j].Manifest.Created)
+	})
+	return sorted[0]
+}
+
+func isFixedPathDestination(destination config.Destination) bool {
+	return destination.PathMap.Mode == config.PathMappingFixed
+}
+
+func fixedPathSelectionWarning(pipelineID, destinationID string, selections []destinationBundleSelection, candidateCount int) OutputWarning {
+	selected := "none"
+	if len(selections) > 0 {
+		selected = storage.DisplayPath(selections[0].SourceBundle.RootRelativePath)
+	}
+	return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
+}
+
+func isDestructiveFixedPathAction(action publish.Action) bool {
+	return action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
+}
+
+func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
+	return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
+}
+
+func destinationIDs(destinations []config.Destination) []string {
+	ids := make([]string, 0, len(destinations))
+	for _, destination := range destinations {
+		ids = append(ids, destination.ID)
+	}
+	return ids
+}

internal/app/run_summary.go

@@ -0,0 +1,74 @@
+package app
+
+import (
+	"fmt"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/publish"
+)
+
+type runSummary struct {
+	dryRun       bool
+	planned      int
+	publishNew   int
+	replaceOlder int
+	forceReplace int
+	skipped      int
+	failures     int
+	fixedPath    int
+}
+
+func (s *runSummary) recordPlan(action publish.Action) {
+	s.planned++
+	switch action {
+	case publish.ActionPublishNew:
+		s.publishNew++
+	case publish.ActionReplaceOlder:
+		s.replaceOlder++
+	case publish.ActionForceReplace:
+		s.forceReplace++
+	case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
+		s.skipped++
+	}
+}
+
+func (s *runSummary) recordFailure() {
+	s.failures++
+}
+
+func (s *runSummary) recordFixedPath() {
+	s.fixedPath++
+}
+
+type RunSummaryCounters struct {
+	Status       string `json:"status"`
+	Planned      int    `json:"planned"`
+	PublishNew   int    `json:"publish_new"`
+	ReplaceOlder int    `json:"replace_older"`
+	ForceReplace int    `json:"force_replace"`
+	Skipped      int    `json:"skipped"`
+	Failed       int    `json:"failed"`
+	DryRun       bool   `json:"dry_run"`
+	FixedPath    int    `json:"fixed_path"`
+}
+
+func (s RunSummaryCounters) Line() string {
+	return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.ReplaceOlder, s.ForceReplace, s.Skipped, s.Failed, s.DryRun, s.FixedPath)
+}
+
+func (s runSummary) Result() RunSummaryCounters {
+	status := "ok"
+	if s.failures > 0 {
+		status = "failed"
+	}
+	return RunSummaryCounters{
+		Status:       status,
+		Planned:      s.planned,
+		PublishNew:   s.publishNew,
+		ReplaceOlder: s.replaceOlder,
+		ForceReplace: s.forceReplace,
+		Skipped:      s.skipped,
+		Failed:       s.failures,
+		DryRun:       s.dryRun,
+		FixedPath:    s.fixedPath,
+	}
+}

internal/app/run_warnings.go

@@ -0,0 +1,47 @@
+package app
+
+import (
+	"fmt"
+	"io"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+)
+
+func secretConflictWarnings(conflicts []config.SecretConflict) []OutputWarning {
+	warnings := make([]OutputWarning, 0, len(conflicts))
+	for _, conflict := range conflicts {
+		warnings = append(warnings, OutputWarning{
+			Message: fmt.Sprintf("secret %s ignored because the real environment already has that variable", conflict.Name),
+		})
+	}
+	return warnings
+}
+
+func sshWarnings(pipeline config.Pipeline) []OutputWarning {
+	var warnings []OutputWarning
+	if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
+		warnings = append(warnings, OutputWarning{
+			Message: fmt.Sprintf("pipeline=%s source host_key_policy=off disables SSH host key checking", pipeline.ID),
+		})
+	}
+	for _, destination := range pipeline.Destinations {
+		if destination.Backend == config.BackendSSH && destination.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
+			warnings = append(warnings, OutputWarning{
+				Message: fmt.Sprintf("pipeline=%s destination=%s host_key_policy=off disables SSH host key checking", pipeline.ID, destination.ID),
+			})
+		}
+	}
+	return warnings
+}
+
+func writeWarnings(w io.Writer, warnings []OutputWarning) error {
+	if w == nil {
+		return nil
+	}
+	for _, warning := range warnings {
+		if _, err := fmt.Fprintf(w, "Warning: %s\n", warning.Message); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/app/runtime.go

@@ -0,0 +1,44 @@
+package app
+
+import "gitea.maximumdirect.net/eric/distributor/internal/config"
+
+type runtimeSetup struct {
+	ConfigPath  string
+	Config      config.Config
+	Environment config.Environment
+	Warnings    []OutputWarning
+}
+
+func loadRuntimeSetup(configPath string) (runtimeSetup, error) {
+	resolvedPath := runtimeConfigPath(configPath)
+	cfg, err := config.LoadFile(resolvedPath)
+	if err != nil {
+		return runtimeSetup{}, err
+	}
+	return runtimeSetupFromConfig(resolvedPath, cfg)
+}
+
+func runtimeSetupFromConfig(configPath string, cfg config.Config) (runtimeSetup, error) {
+	secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
+	if err != nil {
+		return runtimeSetup{}, err
+	}
+	return runtimeSetup{
+		ConfigPath:  configPath,
+		Config:      cfg,
+		Environment: secretLoad.Environment,
+		Warnings:    secretConflictWarnings(secretLoad.Conflicts),
+	}, nil
+}
+
+func runtimeConfigPath(configPath string) string {
+	if configPath == "" {
+		return config.DefaultConfigPath
+	}
+	return configPath
+}
+
+func (setup runtimeSetup) withPipelines(pipelines []config.Pipeline) runtimeSetup {
+	setup.Config.Pipelines = pipelines
+	return setup
+}

internal/app/runtime_test.go

@@ -0,0 +1,32 @@
+package app
+
+import (
+	"testing"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
+)
+
+func TestRuntimeConfigPathDefaultsEmptyPath(t *testing.T) {
+	if got, want := runtimeConfigPath(""), config.DefaultConfigPath; got != want {
+		t.Fatalf("runtimeConfigPath(\"\") = %q, want %q", got, want)
+	}
+	if got, want := runtimeConfigPath("/tmp/distributor.yml"), "/tmp/distributor.yml"; got != want {
+		t.Fatalf("runtimeConfigPath(explicit) = %q, want %q", got, want)
+	}
+}
+
+func TestLoadRuntimeSetupReturnsLoadedConfigPath(t *testing.T) {
+	configPath := testutil.WriteMinimalLocalConfig(t, t.TempDir(), t.TempDir())
+
+	setup, err := loadRuntimeSetup(configPath)
+	if err != nil {
+		t.Fatalf("loadRuntimeSetup() error = %v", err)
+	}
+	if setup.ConfigPath != configPath {
+		t.Fatalf("ConfigPath = %q, want %q", setup.ConfigPath, configPath)
+	}
+	if len(setup.Config.Pipelines) != 1 {
+		t.Fatalf("pipeline count = %d, want 1", len(setup.Config.Pipelines))
+	}
+}

internal/app/serve.go

@@ -0,0 +1,52 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+)
+
+type ServeOptions struct {
+	ConfigPath string
+}
+
+func Serve(ctx context.Context, options ServeOptions) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	setup, err := loadRuntimeSetup(options.ConfigPath)
+	if err != nil {
+		return err
+	}
+
+	handler, err := newUploadHTTPHandler(ctx, setup.Config, setup.Environment)
+	if err != nil {
+		return err
+	}
+	listener, err := net.Listen("tcp", setup.Config.Server.HTTP.Bind)
+	if err != nil {
+		return fmt.Errorf("bind HTTP server %q: %w", setup.Config.Server.HTTP.Bind, err)
+	}
+	defer listener.Close()
+
+	server := &http.Server{Handler: handler}
+	shutdownDone := make(chan struct{})
+	go func() {
+		defer close(shutdownDone)
+		<-ctx.Done()
+		_ = server.Shutdown(context.Background())
+	}()
+
+	err = server.Serve(listener)
+	if errors.Is(err, http.ErrServerClosed) {
+		<-shutdownDone
+		return nil
+	}
+	return err
+}

internal/app/serve_test.go

@@ -0,0 +1,88 @@
+package app
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+func TestServeFailsForUnsafeUploadTokensWithoutLeakingValues(t *testing.T) {
+	duplicateSecret := "duplicate-secret"
+	tests := []struct {
+		name       string
+		configPath func(*testing.T) string
+		env        map[string]string
+		want       string
+	}{
+		{
+			name: "missing token",
+			configPath: func(t *testing.T) string {
+				return writeServeUploadConfig(t, []string{"DISTRIBUTOR_TEST_MISSING_UPLOAD_TOKEN"})
+			},
+			want: "DISTRIBUTOR_TEST_MISSING_UPLOAD_TOKEN",
+		},
+		{
+			name: "empty token",
+			configPath: func(t *testing.T) string {
+				return writeServeUploadConfig(t, []string{"DISTRIBUTOR_TEST_EMPTY_UPLOAD_TOKEN"})
+			},
+			env:  map[string]string{"DISTRIBUTOR_TEST_EMPTY_UPLOAD_TOKEN": ""},
+			want: "DISTRIBUTOR_TEST_EMPTY_UPLOAD_TOKEN",
+		},
+		{
+			name: "duplicate token",
+			configPath: func(t *testing.T) string {
+				return writeServeUploadConfig(t, []string{
+					"DISTRIBUTOR_TEST_FIRST_UPLOAD_TOKEN",
+					"DISTRIBUTOR_TEST_SECOND_UPLOAD_TOKEN",
+				})
+			},
+			env: map[string]string{
+				"DISTRIBUTOR_TEST_FIRST_UPLOAD_TOKEN":  duplicateSecret,
+				"DISTRIBUTOR_TEST_SECOND_UPLOAD_TOKEN": duplicateSecret,
+			},
+			want: "same value",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			for name, value := range tt.env {
+				t.Setenv(name, value)
+			}
+
+			err := Serve(context.Background(), ServeOptions{ConfigPath: tt.configPath(t)})
+			if err == nil {
+				t.Fatal("Serve() error = nil, want token startup error")
+			}
+			if !strings.Contains(err.Error(), tt.want) {
+				t.Fatalf("Serve() error = %v, want %q", err, tt.want)
+			}
+			if strings.Contains(err.Error(), duplicateSecret) {
+				t.Fatalf("Serve() error exposed token value: %v", err)
+			}
+		})
+	}
+}
+
+func writeServeUploadConfig(t *testing.T, tokenEnvs []string) string {
+	t.Helper()
+	body := `
+server:
+  http:
+    bind: 127.0.0.1:0
+pipelines:
+`
+	for index, tokenEnv := range tokenEnvs {
+		body += `
+  - id: reports-` + string(rune('a'+index)) + `
+    source:
+      backend: http_upload
+      token_env: ` + tokenEnv + `
+    destinations:
+      - id: archive
+        backend: local
+        path: ` + t.TempDir() + `
+`
+	}
+	return writeConfigFile(t, body)
+}

internal/app/source_select.go

@@ -0,0 +1,138 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+)
+
+type PipelineNotFoundError struct {
+	ID string
+}
+
+func (e PipelineNotFoundError) Error() string {
+	return fmt.Sprintf("pipeline %q not found", e.ID)
+}
+
+func IsPipelineNotFound(err error) bool {
+	var notFound PipelineNotFoundError
+	return errors.As(err, &notFound)
+}
+
+type sourceCommandOptions struct {
+	CommandName string
+	Path        string
+	ConfigPath  string
+	PipelineID  string
+	BundlePath  string
+}
+
+type sourceSelection struct {
+	Bundles       []bundle.Bundle
+	PipelineID    string
+	SourceBackend string
+	ConfigMode    bool
+	Warnings      []OutputWarning
+}
+
+func selectSourceBundles(ctx context.Context, options sourceCommandOptions, provider backendFactoryProvider) (sourceSelection, error) {
+	if err := ctx.Err(); err != nil {
+		return sourceSelection{}, err
+	}
+	if options.ConfigPath != "" {
+		setup, err := loadRuntimeSetup(options.ConfigPath)
+		if err != nil {
+			return sourceSelection{}, err
+		}
+		return selectSourceBundlesFromSetup(ctx, setup, options, provider)
+	}
+	if options.PipelineID != "" {
+		return sourceSelection{}, fmt.Errorf("configured source mode requires --config")
+	}
+	if options.BundlePath != "" {
+		return sourceSelection{}, fmt.Errorf("configured source mode requires --config")
+	}
+	if options.Path == "" {
+		return sourceSelection{}, fmt.Errorf("%s command requires a path", options.CommandName)
+	}
+	backend, err := newBackendFactory().openLocalPath(ctx, options.Path)
+	if err != nil {
+		return sourceSelection{}, err
+	}
+	defer closeBackend(backend)
+	bundles, err := bundle.Discover(ctx, backend, "")
+	if err != nil {
+		return sourceSelection{}, err
+	}
+	return sourceSelection{Bundles: bundles}, nil
+}
+
+func selectSourceBundlesFromConfig(ctx context.Context, cfg config.Config, options sourceCommandOptions, provider backendFactoryProvider) (sourceSelection, error) {
+	setup, err := runtimeSetupFromConfig("", cfg)
+	if err != nil {
+		return sourceSelection{}, err
+	}
+	return selectSourceBundlesFromSetup(ctx, setup, options, provider)
+}
+
+func selectSourceBundlesFromSetup(ctx context.Context, setup runtimeSetup, options sourceCommandOptions, provider backendFactoryProvider) (sourceSelection, error) {
+	if options.Path != "" {
+		return sourceSelection{}, fmt.Errorf("configured source mode does not accept a local path")
+	}
+	if options.PipelineID == "" {
+		return sourceSelection{}, fmt.Errorf("configured source mode requires --pipeline")
+	}
+	pipeline, ok := findPipeline(setup.Config, options.PipelineID)
+	if !ok {
+		return sourceSelection{}, PipelineNotFoundError{ID: options.PipelineID}
+	}
+	backends := provider(setup.Environment)
+	sourceBackend, err := backends.openSource(ctx, pipeline.Source)
+	if err != nil {
+		return sourceSelection{}, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
+	}
+	defer closeBackend(sourceBackend)
+
+	var bundles []bundle.Bundle
+	if options.BundlePath != "" {
+		sourceBundle, err := bundle.Validate(ctx, sourceBackend, options.BundlePath)
+		if err != nil {
+			return sourceSelection{}, fmt.Errorf("pipeline %s source backend %s bundle %s: %w", pipeline.ID, pipeline.Source.Backend, storage.DisplayPath(options.BundlePath), err)
+		}
+		bundles = []bundle.Bundle{sourceBundle}
+	} else {
+		bundles, err = bundle.Discover(ctx, sourceBackend, "")
+		if err != nil {
+			return sourceSelection{}, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
+		}
+	}
+	return sourceSelection{
+		Bundles:       bundles,
+		PipelineID:    pipeline.ID,
+		SourceBackend: pipeline.Source.Backend,
+		ConfigMode:    true,
+		Warnings:      append(setup.Warnings, sourceSSHWarnings(pipeline)...),
+	}, nil
+}
+
+func findPipeline(cfg config.Config, id string) (config.Pipeline, bool) {
+	for _, pipeline := range cfg.Pipelines {
+		if pipeline.ID == id {
+			return pipeline, true
+		}
+	}
+	return config.Pipeline{}, false
+}
+
+func sourceSSHWarnings(pipeline config.Pipeline) []OutputWarning {
+	if pipeline.Source.Backend != config.BackendSSH || pipeline.Source.SSH.HostKeyPolicy != config.HostKeyPolicyOff {
+		return nil
+	}
+	return []OutputWarning{{
+		Message: fmt.Sprintf("pipeline=%s source host_key_policy=off disables SSH host key checking", pipeline.ID),
+	}}
+}

internal/app/upload_coordinator.go

@@ -0,0 +1,504 @@
+package app
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+	sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
+)
+
+const DefaultUploadMaxFileCount = 4096
+
+type UploadRunID string
+
+type UploadStatus string
+
+const (
+	UploadStatusAccepted  UploadStatus = "accepted"
+	UploadStatusQueued    UploadStatus = "queued"
+	UploadStatusRunning   UploadStatus = "running"
+	UploadStatusSucceeded UploadStatus = "succeeded"
+	UploadStatusFailed    UploadStatus = "failed"
+	UploadStatusExpired   UploadStatus = "expired"
+)
+
+type UploadRunRecord struct {
+	ID         UploadRunID  `json:"run_id"`
+	PipelineID string       `json:"pipeline_id"`
+	Status     UploadStatus `json:"status"`
+	AcceptedAt time.Time    `json:"accepted_at"`
+	StartedAt  *time.Time   `json:"started_at,omitempty"`
+	FinishedAt *time.Time   `json:"finished_at,omitempty"`
+	Report     *RunReport   `json:"report,omitempty"`
+	Error      string       `json:"error,omitempty"`
+	StagedRoot string       `json:"-"`
+}
+
+type UploadRequest struct {
+	PipelineID     string
+	ContentType    string
+	Body           io.Reader
+	IdempotencyKey string
+	DryRun         bool
+	Force          bool
+	MaxFileCount   int
+}
+
+type UploadQueueFullError struct {
+	QueueSize int
+}
+
+func (err UploadQueueFullError) Error() string {
+	return fmt.Sprintf("upload queue is full with capacity %d", err.QueueSize)
+}
+
+func IsUploadQueueFull(err error) bool {
+	var full UploadQueueFullError
+	return errors.As(err, &full)
+}
+
+type UploadIdempotencyConflictError struct {
+	Retryable bool
+}
+
+func (err UploadIdempotencyConflictError) Error() string {
+	if err.Retryable {
+		return "upload idempotency key is already being processed"
+	}
+	return "upload idempotency key conflicts with a different source manifest"
+}
+
+func IsUploadIdempotencyConflict(err error) bool {
+	var conflict UploadIdempotencyConflictError
+	return errors.As(err, &conflict)
+}
+
+type UploadCoordinator struct {
+	ctx          context.Context
+	cfg          config.Config
+	stage        uploadStageFunc
+	run          uploadRunFunc
+	now          func() time.Time
+	randomSuffix func() (string, error)
+	retention    time.Duration
+
+	mu             sync.Mutex
+	signal         chan struct{}
+	queueSize      int
+	maxConcurrency int
+	runningCount   int
+	reservedCount  int
+	activePipeline map[string]bool
+	pending        []*uploadJob
+	records        map[UploadRunID]UploadRunRecord
+	idempotency    map[uploadIdempotencyScope]uploadIdempotencyRecord
+}
+
+type uploadStageFunc func(context.Context, ingest.StageOptions) (ingest.StagedBundle, error)
+
+type uploadRunFunc func(context.Context, config.Config, RunPipelineWithLocalSourceOptions) (RunReport, error)
+
+type uploadJob struct {
+	recordID   UploadRunID
+	request    UploadRequest
+	pipeline   config.Pipeline
+	stagedRoot string
+}
+
+type uploadIdempotencyScope struct {
+	PipelineID string
+	Key        string
+}
+
+type uploadIdempotencyRecord struct {
+	RunID    UploadRunID
+	Manifest sourcebundle.Manifest
+	Pending  bool
+}
+
+type uploadCoordinatorHooks struct {
+	stage        uploadStageFunc
+	run          uploadRunFunc
+	now          func() time.Time
+	randomSuffix func() (string, error)
+}
+
+func NewUploadCoordinator(ctx context.Context, cfg config.Config) *UploadCoordinator {
+	return newUploadCoordinator(ctx, cfg, uploadCoordinatorHooks{})
+}
+
+func newUploadCoordinator(ctx context.Context, cfg config.Config, hooks uploadCoordinatorHooks) *UploadCoordinator {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	config.ApplyDefaults(&cfg)
+	stage := hooks.stage
+	if stage == nil {
+		stage = ingest.StageArchive
+	}
+	run := hooks.run
+	if run == nil {
+		run = runPipelineConfigWithLocalSource
+	}
+	now := hooks.now
+	if now == nil {
+		now = time.Now
+	}
+	randomSuffix := hooks.randomSuffix
+	if randomSuffix == nil {
+		randomSuffix = randomRunIDSuffix
+	}
+	coordinator := &UploadCoordinator{
+		ctx:            ctx,
+		cfg:            cfg,
+		stage:          stage,
+		run:            run,
+		now:            now,
+		randomSuffix:   randomSuffix,
+		retention:      cfg.Server.HTTP.Retention.AsDuration(),
+		signal:         make(chan struct{}, 1),
+		queueSize:      cfg.Server.HTTP.QueueSize,
+		maxConcurrency: cfg.Server.HTTP.MaxConcurrency,
+		activePipeline: map[string]bool{},
+		records:        map[UploadRunID]UploadRunRecord{},
+		idempotency:    map[uploadIdempotencyScope]uploadIdempotencyRecord{},
+	}
+	go coordinator.dispatchLoop()
+	return coordinator
+}
+
+func (coordinator *UploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := ctx.Err(); err != nil {
+		return UploadRunRecord{}, err
+	}
+	if request.Body == nil {
+		return UploadRunRecord{}, fmt.Errorf("upload body is required")
+	}
+	pipeline, ok := findPipeline(coordinator.cfg, request.PipelineID)
+	if !ok {
+		return UploadRunRecord{}, PipelineNotFoundError{ID: request.PipelineID}
+	}
+	if pipeline.Source.Backend != config.BackendHTTPUpload {
+		return UploadRunRecord{}, fmt.Errorf("pipeline %s source backend %s is not configured for uploads", pipeline.ID, pipeline.Source.Backend)
+	}
+	runID, err := coordinator.newRunID(pipeline.ID)
+	if err != nil {
+		return UploadRunRecord{}, err
+	}
+	if err := ingest.ValidateContentType(request.ContentType); err != nil {
+		return UploadRunRecord{}, err
+	}
+	scope, hasKey := uploadRequestIdempotencyScope(pipeline.ID, request.IdempotencyKey)
+
+	coordinator.mu.Lock()
+	coordinator.expireLocked(coordinator.now().UTC())
+	existingIdempotency, hasExistingIdempotency := coordinator.idempotency[scope]
+	if hasKey && hasExistingIdempotency && existingIdempotency.Pending {
+		coordinator.mu.Unlock()
+		return UploadRunRecord{}, UploadIdempotencyConflictError{Retryable: true}
+	}
+	needsReservation := !hasKey || !hasExistingIdempotency
+	if needsReservation {
+		if coordinator.queueFullLocked() {
+			coordinator.mu.Unlock()
+			return UploadRunRecord{}, UploadQueueFullError{QueueSize: coordinator.queueSize}
+		}
+		coordinator.reservedCount++
+		if hasKey {
+			coordinator.idempotency[scope] = uploadIdempotencyRecord{Pending: true}
+		}
+	}
+	coordinator.mu.Unlock()
+
+	staged, err := coordinator.stage(ctx, ingest.StageOptions{
+		Body:                request.Body,
+		ContentType:         request.ContentType,
+		PipelineStagingPath: pipeline.Source.Upload.StagingPath,
+		RunID:               string(runID),
+		MaxUploadSize:       int64(*pipeline.Source.Upload.MaxUploadSize),
+		MaxExtractedSize:    int64(*pipeline.Source.Upload.MaxUploadSize),
+		MaxFileCount:        uploadMaxFileCount(request.MaxFileCount),
+	})
+	if err != nil {
+		if needsReservation {
+			coordinator.releaseReservation(scope, hasKey)
+		}
+		return UploadRunRecord{}, err
+	}
+
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	if needsReservation {
+		coordinator.reservedCount--
+	}
+	if hasKey {
+		existingIdempotency, hasExistingIdempotency = coordinator.idempotency[scope]
+		if hasExistingIdempotency && !existingIdempotency.Pending {
+			if uploadManifestsEqual(existingIdempotency.Manifest, staged.Manifest) {
+				_ = os.RemoveAll(staged.Root)
+				record, ok := coordinator.records[existingIdempotency.RunID]
+				if !ok {
+					return UploadRunRecord{}, fmt.Errorf("idempotency record references missing run")
+				}
+				return record, nil
+			}
+			_ = os.RemoveAll(staged.Root)
+			return UploadRunRecord{}, UploadIdempotencyConflictError{}
+		}
+		if !hasExistingIdempotency && !needsReservation && coordinator.queueFullLocked() {
+			_ = os.RemoveAll(staged.Root)
+			return UploadRunRecord{}, UploadQueueFullError{QueueSize: coordinator.queueSize}
+		}
+	}
+	record := UploadRunRecord{
+		ID:         runID,
+		PipelineID: pipeline.ID,
+		Status:     UploadStatusAccepted,
+		AcceptedAt: coordinator.now().UTC(),
+		StagedRoot: staged.Root,
+	}
+	coordinator.records[runID] = record
+	if hasKey {
+		coordinator.idempotency[scope] = uploadIdempotencyRecord{
+			RunID:    runID,
+			Manifest: staged.Manifest,
+		}
+	}
+	coordinator.pending = append(coordinator.pending, &uploadJob{
+		recordID:   runID,
+		request:    request,
+		pipeline:   pipeline,
+		stagedRoot: staged.Root,
+	})
+	coordinator.notify()
+	return record, nil
+}
+
+func uploadRequestIdempotencyScope(pipelineID, key string) (uploadIdempotencyScope, bool) {
+	if key == "" {
+		return uploadIdempotencyScope{}, false
+	}
+	return uploadIdempotencyScope{PipelineID: pipelineID, Key: key}, true
+}
+
+func (coordinator *UploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	coordinator.expireLocked(coordinator.now().UTC())
+	record, ok := coordinator.records[runID]
+	return record, ok
+}
+
+func (coordinator *UploadCoordinator) Expire() []UploadRunRecord {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	return coordinator.expireLocked(coordinator.now().UTC())
+}
+
+func (coordinator *UploadCoordinator) CanAccept() bool {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	coordinator.expireLocked(coordinator.now().UTC())
+	return !coordinator.queueFullLocked()
+}
+
+func (coordinator *UploadCoordinator) QueueDepth() int {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	return len(coordinator.pending) + coordinator.reservedCount
+}
+
+func (coordinator *UploadCoordinator) RunningCount() int {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	return coordinator.runningCount
+}
+
+func (coordinator *UploadCoordinator) newRunID(pipelineID string) (UploadRunID, error) {
+	suffix, err := coordinator.randomSuffix()
+	if err != nil {
+		return "", err
+	}
+	timestamp := coordinator.now().UTC().Format("20060102T150405Z")
+	return UploadRunID(pipelineID + "." + timestamp + "." + suffix), nil
+}
+
+func (coordinator *UploadCoordinator) dispatchLoop() {
+	for {
+		select {
+		case <-coordinator.ctx.Done():
+			return
+		case <-coordinator.signal:
+			for coordinator.startNext() {
+			}
+		}
+	}
+}
+
+func (coordinator *UploadCoordinator) startNext() bool {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	if coordinator.runningCount >= coordinator.maxConcurrency {
+		coordinator.markPendingQueuedLocked()
+		return false
+	}
+	index := -1
+	for candidateIndex, job := range coordinator.pending {
+		if coordinator.activePipeline[job.pipeline.ID] {
+			record := coordinator.records[job.recordID]
+			if record.Status == UploadStatusAccepted {
+				record.Status = UploadStatusQueued
+				coordinator.records[job.recordID] = record
+			}
+			continue
+		}
+		index = candidateIndex
+		break
+	}
+	if index < 0 {
+		return false
+	}
+
+	job := coordinator.pending[index]
+	coordinator.pending = append(coordinator.pending[:index], coordinator.pending[index+1:]...)
+	now := coordinator.now().UTC()
+	record := coordinator.records[job.recordID]
+	record.Status = UploadStatusRunning
+	record.StartedAt = &now
+	coordinator.records[job.recordID] = record
+	coordinator.runningCount++
+	coordinator.activePipeline[job.pipeline.ID] = true
+
+	go coordinator.runJob(job)
+	return true
+}
+
+func (coordinator *UploadCoordinator) markPendingQueuedLocked() {
+	for _, job := range coordinator.pending {
+		record := coordinator.records[job.recordID]
+		if record.Status == UploadStatusAccepted {
+			record.Status = UploadStatusQueued
+			coordinator.records[job.recordID] = record
+		}
+	}
+}
+
+func (coordinator *UploadCoordinator) runJob(job *uploadJob) {
+	report, err := coordinator.run(coordinator.ctx, coordinator.cfg, RunPipelineWithLocalSourceOptions{
+		PipelineID: job.pipeline.ID,
+		SourceRoot: job.stagedRoot,
+		DryRun:     job.request.DryRun,
+		Force:      job.request.Force,
+	})
+	coordinator.complete(job, &report, err)
+}
+
+func (coordinator *UploadCoordinator) releaseReservation(scope uploadIdempotencyScope, hasKey bool) {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	coordinator.reservedCount--
+	if hasKey {
+		if record, ok := coordinator.idempotency[scope]; ok && record.Pending {
+			delete(coordinator.idempotency, scope)
+		}
+	}
+}
+
+func (coordinator *UploadCoordinator) queueFullLocked() bool {
+	return len(coordinator.pending)+coordinator.reservedCount >= coordinator.queueSize
+}
+
+func uploadMaxFileCount(value int) int {
+	if value > 0 {
+		return value
+	}
+	return DefaultUploadMaxFileCount
+}
+
+func (coordinator *UploadCoordinator) complete(job *uploadJob, report *RunReport, runErr error) {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	record := coordinator.records[job.recordID]
+	finishedAt := coordinator.now().UTC()
+	record.FinishedAt = &finishedAt
+	record.Report = report
+	if runErr != nil {
+		record.Status = UploadStatusFailed
+		record.Error = runErr.Error()
+	} else {
+		record.Status = UploadStatusSucceeded
+	}
+	coordinator.records[job.recordID] = record
+	coordinator.runningCount--
+	delete(coordinator.activePipeline, job.pipeline.ID)
+	coordinator.notify()
+}
+
+func (coordinator *UploadCoordinator) expireLocked(now time.Time) []UploadRunRecord {
+	var expired []UploadRunRecord
+	for runID, record := range coordinator.records {
+		if record.FinishedAt == nil || record.Status == UploadStatusExpired {
+			continue
+		}
+		if now.Before(record.FinishedAt.Add(coordinator.retention)) {
+			continue
+		}
+		if record.StagedRoot != "" {
+			_ = os.RemoveAll(record.StagedRoot)
+		}
+		record.Status = UploadStatusExpired
+		record.Report = nil
+		record.Error = ""
+		expired = append(expired, record)
+		delete(coordinator.records, runID)
+		for scope, idempotencyRecord := range coordinator.idempotency {
+			if idempotencyRecord.RunID == runID {
+				delete(coordinator.idempotency, scope)
+			}
+		}
+	}
+	return expired
+}
+
+func uploadManifestsEqual(a, b sourcebundle.Manifest) bool {
+	if a.SchemaVersion != b.SchemaVersion ||
+		a.ID != b.ID ||
+		a.Digest != b.Digest ||
+		!a.Created.Equal(b.Created) ||
+		len(a.Files) != len(b.Files) {
+		return false
+	}
+	for index := range a.Files {
+		if a.Files[index] != b.Files[index] {
+			return false
+		}
+	}
+	return true
+}
+
+func (coordinator *UploadCoordinator) notify() {
+	select {
+	case coordinator.signal <- struct{}{}:
+	default:
+	}
+}
+
+func randomRunIDSuffix() (string, error) {
+	var data [4]byte
+	if _, err := rand.Read(data[:]); err != nil {
+		return "", fmt.Errorf("generate run id suffix: %w", err)
+	}
+	return hex.EncodeToString(data[:]), nil
+}

internal/app/upload_coordinator_test.go

@@ -0,0 +1,632 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+	sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
+)
+
+func TestUploadCoordinatorGeneratesRunIDAndAcceptedStatus(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	clock := newUploadTestClock(time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC))
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"weather-daily"},
+	}), uploadCoordinatorHooks{
+		now:          clock.Now,
+		randomSuffix: uploadTestSuffixes("ab12cd34"),
+		stage:        successfulUploadStage,
+		run:          successfulUploadRun,
+	})
+
+	record, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "weather-daily",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("archive"),
+	})
+	if err != nil {
+		t.Fatalf("Submit() error = %v", err)
+	}
+	if got, want := record.ID, UploadRunID("weather-daily.20260603T120000Z.ab12cd34"); got != want {
+		t.Fatalf("run id = %q, want %q", got, want)
+	}
+	if got, want := record.Status, UploadStatusAccepted; got != want {
+		t.Fatalf("initial status = %q, want %q", got, want)
+	}
+	waitForUploadStatus(t, coordinator, record.ID, UploadStatusSucceeded)
+}
+
+func TestUploadCoordinatorRejectsFullQueueBeforeReadingBody(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	var reads atomic.Int64
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs:    []string{"reports"},
+		queueSize:      1,
+		maxConcurrency: 1,
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002", "00000003"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			<-release
+			return RunReport{}, nil
+		},
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("first"),
+	})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
+
+	second, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("second"),
+	})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusQueued)
+
+	_, err = coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body: readerFunc(func(data []byte) (int, error) {
+			reads.Add(1)
+			return 0, io.EOF
+		}),
+	})
+	if err == nil || !IsUploadQueueFull(err) {
+		t.Fatalf("third Submit() error = %v, want full queue", err)
+	}
+	if got := reads.Load(); got != 0 {
+		t.Fatalf("rejected body reads = %d, want 0", got)
+	}
+	close(release)
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
+}
+
+func TestUploadCoordinatorSerializesSamePipelineUploads(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs:    []string{"reports"},
+		queueSize:      4,
+		maxConcurrency: 2,
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			<-release
+			return RunReport{}, nil
+		},
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("first")})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	second, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("second")})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusQueued)
+	if got := coordinator.RunningCount(); got != 1 {
+		t.Fatalf("running count = %d, want 1", got)
+	}
+	close(release)
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
+}
+
+func TestUploadCoordinatorRunsDifferentPipelinesConcurrentlyUpToLimit(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs:    []string{"reports-one", "reports-two"},
+		queueSize:      4,
+		maxConcurrency: 2,
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			<-release
+			return RunReport{}, nil
+		},
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports-one", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("first")})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	second, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports-two", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("second")})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusRunning)
+	if got := coordinator.RunningCount(); got != 2 {
+		t.Fatalf("running count = %d, want 2", got)
+	}
+	close(release)
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
+}
+
+func TestUploadCoordinatorRecordsFailureDetails(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	runErr := errors.New("publish failed")
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			return RunReport{DryRun: options.DryRun}, runErr
+		},
+	})
+
+	record, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("archive"),
+		DryRun:      true,
+	})
+	if err != nil {
+		t.Fatalf("Submit() error = %v", err)
+	}
+	failed := waitForUploadStatus(t, coordinator, record.ID, UploadStatusFailed)
+	if failed.Error != runErr.Error() {
+		t.Fatalf("error = %q, want %q", failed.Error, runErr.Error())
+	}
+	if failed.Report == nil || !failed.Report.DryRun {
+		t.Fatalf("report = %#v, want retained dry-run report", failed.Report)
+	}
+}
+
+func TestUploadCoordinatorExpiresCompletedRecordsAndStagingDirectories(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	clock := newUploadTestClock(time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC))
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+		retention:   time.Second,
+	}), uploadCoordinatorHooks{
+		now:          clock.Now,
+		randomSuffix: uploadTestSuffixes("00000001"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			return RunReport{DryRun: true}, nil
+		},
+	})
+
+	record, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("archive"),
+	})
+	if err != nil {
+		t.Fatalf("Submit() error = %v", err)
+	}
+	succeeded := waitForUploadStatus(t, coordinator, record.ID, UploadStatusSucceeded)
+	if succeeded.Report == nil || !succeeded.Report.DryRun {
+		t.Fatalf("report = %#v, want retained dry-run report", succeeded.Report)
+	}
+	if _, err := os.Stat(succeeded.StagedRoot); err != nil {
+		t.Fatalf("staged root stat before expiry = %v", err)
+	}
+
+	clock.Advance(2 * time.Second)
+	expired := coordinator.Expire()
+	if got, want := len(expired), 1; got != want {
+		t.Fatalf("expired count = %d, want %d", got, want)
+	}
+	if expired[0].Status != UploadStatusExpired || expired[0].Report != nil || expired[0].Error != "" {
+		t.Fatalf("expired record = %#v, want expired without report/error", expired[0])
+	}
+	if _, ok := coordinator.Status(record.ID); ok {
+		t.Fatal("Status() ok = true after expiry, want removed status")
+	}
+	if _, err := os.Stat(succeeded.StagedRoot); !os.IsNotExist(err) {
+		t.Fatalf("staged root stat after expiry = %v, want not exist", err)
+	}
+}
+
+func TestUploadCoordinatorIdempotencyReturnsOriginalRunForSameManifest(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	var runCount atomic.Int64
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        manifestUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			runCount.Add(1)
+			return RunReport{}, nil
+		},
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:     "reports",
+		ContentType:    ingest.ContentTypeTar,
+		Body:           strings.NewReader("same"),
+		IdempotencyKey: "producer.retry:20260603",
+	})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+
+	second, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:     "reports",
+		ContentType:    ingest.ContentTypeTar,
+		Body:           strings.NewReader("same"),
+		IdempotencyKey: "producer.retry:20260603",
+	})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	if second.ID != first.ID {
+		t.Fatalf("second run id = %q, want original %q", second.ID, first.ID)
+	}
+	if got := runCount.Load(); got != 1 {
+		t.Fatalf("run count = %d, want 1", got)
+	}
+}
+
+func TestUploadCoordinatorIdempotencyConflictsForDifferentManifest(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        manifestUploadStage,
+		run:          successfulUploadRun,
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:     "reports",
+		ContentType:    ingest.ContentTypeTar,
+		Body:           strings.NewReader("one"),
+		IdempotencyKey: "same-key",
+	})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+
+	_, err = coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:     "reports",
+		ContentType:    ingest.ContentTypeTar,
+		Body:           strings.NewReader("two"),
+		IdempotencyKey: "same-key",
+	})
+	if err == nil || !IsUploadIdempotencyConflict(err) {
+		t.Fatalf("second Submit() error = %v, want idempotency conflict", err)
+	}
+}
+
+func TestUploadCoordinatorIdempotencyIsScopedByPipeline(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports-one", "reports-two"},
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        manifestUploadStage,
+		run:          successfulUploadRun,
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:     "reports-one",
+		ContentType:    ingest.ContentTypeTar,
+		Body:           strings.NewReader("one"),
+		IdempotencyKey: "shared-key",
+	})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	second, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:     "reports-two",
+		ContentType:    ingest.ContentTypeTar,
+		Body:           strings.NewReader("two"),
+		IdempotencyKey: "shared-key",
+	})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	if second.ID == first.ID {
+		t.Fatalf("run ids matched across pipelines: %q", second.ID)
+	}
+}
+
+func TestUploadCoordinatorWithoutIdempotencyKeyAcceptsDuplicateBodies(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        manifestUploadStage,
+		run:          successfulUploadRun,
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("same")})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	second, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("same")})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	if second.ID == first.ID {
+		t.Fatalf("second run id = %q, want distinct run", second.ID)
+	}
+}
+
+func TestUploadCoordinatorIdempotencyReturnsRetryableConflictWhileStaging(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage: func(ctx context.Context, opts ingest.StageOptions) (ingest.StagedBundle, error) {
+			close(entered)
+			<-release
+			return manifestUploadStage(ctx, opts)
+		},
+		run: successfulUploadRun,
+	})
+	firstErr := make(chan error, 1)
+	go func() {
+		_, err := coordinator.Submit(context.Background(), UploadRequest{
+			PipelineID:     "reports",
+			ContentType:    ingest.ContentTypeTar,
+			Body:           strings.NewReader("same"),
+			IdempotencyKey: "in-flight",
+		})
+		firstErr <- err
+	}()
+	<-entered
+
+	var reads atomic.Int64
+	_, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body: readerFunc(func(data []byte) (int, error) {
+			reads.Add(1)
+			return 0, io.EOF
+		}),
+		IdempotencyKey: "in-flight",
+	})
+	var conflict UploadIdempotencyConflictError
+	if err == nil || !errors.As(err, &conflict) || !conflict.Retryable {
+		t.Fatalf("second Submit() error = %v, want retryable idempotency conflict", err)
+	}
+	if got := reads.Load(); got != 0 {
+		t.Fatalf("retryable conflict body reads = %d, want 0", got)
+	}
+	close(release)
+	if err := <-firstErr; err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+}
+
+func TestUploadCoordinatorIdempotencyExpiresWithCompletedStatus(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	clock := newUploadTestClock(time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC))
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+		retention:   time.Second,
+	}), uploadCoordinatorHooks{
+		now:          clock.Now,
+		randomSuffix: uploadTestSuffixes("00000001", "00000002", "00000003"),
+		stage:        manifestUploadStage,
+		run:          successfulUploadRun,
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:     "reports",
+		ContentType:    ingest.ContentTypeTar,
+		Body:           strings.NewReader("same"),
+		IdempotencyKey: "expires",
+	})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+
+	clock.Advance(2 * time.Second)
+	coordinator.Expire()
+
+	second, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:     "reports",
+		ContentType:    ingest.ContentTypeTar,
+		Body:           strings.NewReader("same"),
+		IdempotencyKey: "expires",
+	})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	if second.ID == first.ID {
+		t.Fatalf("second run id = %q, want new run after expiry", second.ID)
+	}
+}
+
+type readerFunc func([]byte) (int, error)
+
+func (fn readerFunc) Read(data []byte) (int, error) {
+	return fn(data)
+}
+
+func successfulUploadStage(ctx context.Context, opts ingest.StageOptions) (ingest.StagedBundle, error) {
+	root := filepath.Join(opts.PipelineStagingPath, opts.RunID)
+	if err := os.MkdirAll(root, 0o755); err != nil {
+		return ingest.StagedBundle{}, err
+	}
+	return ingest.StagedBundle{Root: root}, nil
+}
+
+func manifestUploadStage(ctx context.Context, opts ingest.StageOptions) (ingest.StagedBundle, error) {
+	data, err := io.ReadAll(opts.Body)
+	if err != nil {
+		return ingest.StagedBundle{}, err
+	}
+	root := filepath.Join(opts.PipelineStagingPath, opts.RunID)
+	if err := os.MkdirAll(root, 0o755); err != nil {
+		return ingest.StagedBundle{}, err
+	}
+	return ingest.StagedBundle{
+		Root:     root,
+		Manifest: uploadTestManifest(string(data)),
+	}, nil
+}
+
+func uploadTestManifest(id string) sourcebundle.Manifest {
+	created := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
+	file := sourcebundle.ManifestFile{
+		Path:   "report.md",
+		SHA256: sourcebundle.FileDigest([]byte(id)),
+		Size:   int64(len(id)),
+	}
+	return sourcebundle.Manifest{
+		SchemaVersion: sourcebundle.SchemaVersion,
+		ID:            id,
+		Created:       created,
+		Files:         []sourcebundle.ManifestFile{file},
+		Digest:        sourcebundle.BundleDigest([]sourcebundle.ManifestFile{file}),
+	}
+}
+
+func successfulUploadRun(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+	return RunReport{}, nil
+}
+
+type uploadCoordinatorConfigOptions struct {
+	pipelineIDs    []string
+	queueSize      int
+	maxConcurrency int
+	retention      time.Duration
+}
+
+func uploadCoordinatorConfig(t *testing.T, opts uploadCoordinatorConfigOptions) config.Config {
+	t.Helper()
+	queueSize := opts.queueSize
+	if queueSize == 0 {
+		queueSize = 4
+	}
+	maxConcurrency := opts.maxConcurrency
+	if maxConcurrency == 0 {
+		maxConcurrency = 1
+	}
+	retentionValue := opts.retention
+	if retentionValue == 0 {
+		retentionValue = time.Minute
+	}
+	retention := config.Duration(retentionValue)
+	maxUploadSize := config.ByteSize(1024)
+	cfg := config.Config{
+		Server: config.Server{HTTP: config.HTTPServer{
+			StagingRoot:    t.TempDir(),
+			MaxUploadSize:  &maxUploadSize,
+			QueueSize:      queueSize,
+			MaxConcurrency: maxConcurrency,
+			Retention:      &retention,
+		}},
+	}
+	for _, pipelineID := range opts.pipelineIDs {
+		cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
+			ID: pipelineID,
+			Source: config.Backend{
+				Backend: config.BackendHTTPUpload,
+				Upload:  config.HTTPUpload{TokenEnv: strings.ToUpper(strings.ReplaceAll(pipelineID, "-", "_")) + "_TOKEN"},
+			},
+			Destinations: []config.Destination{{
+				ID:      "archive",
+				Backend: config.BackendLocal,
+				Path:    t.TempDir(),
+			}},
+		})
+	}
+	return cfg
+}
+
+type uploadTestClock struct {
+	mu  sync.Mutex
+	now time.Time
+}
+
+func newUploadTestClock(now time.Time) *uploadTestClock {
+	return &uploadTestClock{now: now}
+}
+
+func (clock *uploadTestClock) Now() time.Time {
+	clock.mu.Lock()
+	defer clock.mu.Unlock()
+	return clock.now
+}
+
+func (clock *uploadTestClock) Advance(duration time.Duration) {
+	clock.mu.Lock()
+	defer clock.mu.Unlock()
+	clock.now = clock.now.Add(duration)
+}
+
+func uploadTestSuffixes(values ...string) func() (string, error) {
+	var mu sync.Mutex
+	index := 0
+	return func() (string, error) {
+		mu.Lock()
+		defer mu.Unlock()
+		if index >= len(values) {
+			return fmt.Sprintf("%08d", index+1), nil
+		}
+		value := values[index]
+		index++
+		return value, nil
+	}
+}
+
+func waitForUploadStatus(t *testing.T, coordinator *UploadCoordinator, runID UploadRunID, status UploadStatus) UploadRunRecord {
+	t.Helper()
+	deadline := time.Now().Add(time.Second)
+	for time.Now().Before(deadline) {
+		record, ok := coordinator.Status(runID)
+		if ok && record.Status == status {
+			return record
+		}
+		time.Sleep(time.Millisecond)
+	}
+	record, ok := coordinator.Status(runID)
+	t.Fatalf("timed out waiting for status %s; latest ok=%t record=%#v", status, ok, record)
+	return UploadRunRecord{}
+}

internal/app/upload_http.go

@@ -0,0 +1,211 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+)
+
+type uploadCoordinator interface {
+	CanAccept() bool
+	Submit(context.Context, UploadRequest) (UploadRunRecord, error)
+	Status(UploadRunID) (UploadRunRecord, bool)
+}
+
+type uploadHTTPHandler struct {
+	coordinator uploadCoordinator
+	tokens      map[string]string
+}
+
+type uploadAcceptedResponse struct {
+	RunID  UploadRunID  `json:"run_id"`
+	Status UploadStatus `json:"status"`
+}
+
+type httpErrorResponse struct {
+	Error     string `json:"error"`
+	Retryable bool   `json:"retryable,omitempty"`
+}
+
+const idempotencyKeyHeader = "Idempotency-Key"
+
+func newUploadHTTPHandler(ctx context.Context, cfg config.Config, environment config.Environment) (http.Handler, error) {
+	config.ApplyDefaults(&cfg)
+	tokens, err := resolveUploadTokens(cfg, environment)
+	if err != nil {
+		return nil, err
+	}
+	return uploadHTTPHandler{
+		coordinator: NewUploadCoordinator(ctx, cfg),
+		tokens:      tokens,
+	}, nil
+}
+
+func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]string, error) {
+	tokens := make(map[string]string)
+	for _, pipeline := range cfg.Pipelines {
+		if pipeline.Source.Backend != config.BackendHTTPUpload {
+			continue
+		}
+		tokenName := pipeline.Source.Upload.TokenEnv
+		token, ok := environment.Lookup(tokenName)
+		if !ok {
+			return nil, fmt.Errorf("upload token environment variable %s is not set", tokenName)
+		}
+		if token == "" {
+			return nil, fmt.Errorf("upload token environment variable %s is empty", tokenName)
+		}
+		if existing, exists := tokens[token]; exists {
+			return nil, fmt.Errorf("upload token environment variables for pipelines %s and %s resolve to the same value", existing, pipeline.ID)
+		}
+		tokens[token] = pipeline.ID
+	}
+	return tokens, nil
+}
+
+func (handler uploadHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	switch {
+	case r.Method == http.MethodGet && r.URL.Path == "/healthz":
+		handler.handleHealth(w)
+	case r.Method == http.MethodPost && r.URL.Path == "/upload":
+		handler.handleUpload(w, r)
+	case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/runs/"):
+		handler.handleRunStatus(w, r)
+	default:
+		writeHTTPError(w, http.StatusNotFound, "not found")
+	}
+}
+
+func (handler uploadHTTPHandler) handleHealth(w http.ResponseWriter) {
+	writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+}
+
+func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Query().Has("pipeline") || r.URL.Query().Has("pipeline_id") {
+		writeHTTPError(w, http.StatusBadRequest, "pipeline id is not accepted")
+		return
+	}
+	pipelineID, ok := handler.authenticate(r.Header.Get("Authorization"))
+	if !ok {
+		writeHTTPError(w, http.StatusUnauthorized, "unauthorized")
+		return
+	}
+	contentType := r.Header.Get("Content-Type")
+	if err := ingest.ValidateContentType(contentType); err != nil {
+		writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
+		return
+	}
+	idempotencyKey, err := uploadIdempotencyKey(r.Header)
+	if err != nil {
+		writeHTTPError(w, http.StatusBadRequest, "invalid idempotency key")
+		return
+	}
+	record, err := handler.coordinator.Submit(r.Context(), UploadRequest{
+		PipelineID:     pipelineID,
+		ContentType:    contentType,
+		Body:           r.Body,
+		IdempotencyKey: idempotencyKey,
+	})
+	if err != nil {
+		writeUploadSubmitError(w, err)
+		return
+	}
+	writeJSON(w, http.StatusAccepted, uploadAcceptedResponse{
+		RunID:  record.ID,
+		Status: UploadStatusAccepted,
+	})
+}
+
+func (handler uploadHTTPHandler) handleRunStatus(w http.ResponseWriter, r *http.Request) {
+	rawRunID := strings.TrimPrefix(r.URL.Path, "/runs/")
+	if rawRunID == "" || strings.Contains(rawRunID, "/") {
+		writeHTTPError(w, http.StatusNotFound, "not found")
+		return
+	}
+	record, ok := handler.coordinator.Status(UploadRunID(rawRunID))
+	if !ok {
+		writeHTTPError(w, http.StatusNotFound, "run not found")
+		return
+	}
+	writeJSON(w, http.StatusOK, record)
+}
+
+func (handler uploadHTTPHandler) authenticate(header string) (string, bool) {
+	const prefix = "Bearer "
+	if !strings.HasPrefix(header, prefix) {
+		return "", false
+	}
+	token := strings.TrimSpace(strings.TrimPrefix(header, prefix))
+	if token == "" {
+		return "", false
+	}
+	pipelineID, ok := handler.tokens[token]
+	return pipelineID, ok
+}
+
+func uploadIdempotencyKey(header http.Header) (string, error) {
+	values := header.Values(idempotencyKeyHeader)
+	if len(values) == 0 {
+		return "", nil
+	}
+	if len(values) != 1 {
+		return "", fmt.Errorf("idempotency key must appear at most once")
+	}
+	key := values[0]
+	if key == "" {
+		return "", fmt.Errorf("idempotency key is required when header is present")
+	}
+	if len(key) > 128 {
+		return "", fmt.Errorf("idempotency key must be at most 128 bytes")
+	}
+	for index := 0; index < len(key); index++ {
+		character := key[index]
+		if character >= 'a' && character <= 'z' ||
+			character >= 'A' && character <= 'Z' ||
+			character >= '0' && character <= '9' ||
+			character == '.' ||
+			character == '_' ||
+			character == '-' ||
+			character == ':' {
+			continue
+		}
+		return "", fmt.Errorf("idempotency key contains unsupported character")
+	}
+	return key, nil
+}
+
+func writeUploadSubmitError(w http.ResponseWriter, err error) {
+	var idempotencyConflict UploadIdempotencyConflictError
+	switch {
+	case IsUploadQueueFull(err):
+		writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
+	case errors.As(err, &idempotencyConflict):
+		writeHTTPErrorRetryable(w, http.StatusConflict, idempotencyConflict.Error(), idempotencyConflict.Retryable)
+	case errors.Is(err, ingest.ErrUploadTooLarge):
+		writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
+	case errors.Is(err, ingest.ErrUnsupportedContentType):
+		writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
+	default:
+		writeHTTPError(w, http.StatusBadRequest, "upload rejected")
+	}
+}
+
+func writeHTTPError(w http.ResponseWriter, status int, message string) {
+	writeHTTPErrorRetryable(w, status, message, false)
+}
+
+func writeHTTPErrorRetryable(w http.ResponseWriter, status int, message string, retryable bool) {
+	writeJSON(w, status, httpErrorResponse{Error: message, Retryable: retryable})
+}
+
+func writeJSON(w http.ResponseWriter, status int, value any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(value)
+}

internal/app/upload_http_integration_test.go

@@ -0,0 +1,533 @@
+package app
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
+)
+
+func TestHTTPUploadPublishesTarAndGzipFanout(t *testing.T) {
+	tests := []struct {
+		name        string
+		compressed  bool
+		contentType string
+	}{
+		{name: "tar", contentType: ingest.ContentTypeTar},
+		{name: "gzip", compressed: true, contentType: ingest.ContentTypeGzip},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			firstDestination := t.TempDir()
+			secondDestination := t.TempDir()
+			cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+				id:           "reports",
+				tokenEnv:     "REPORTS_TOKEN",
+				stagingPath:  filepath.Join(t.TempDir(), "reports"),
+				destinations: []string{firstDestination, secondDestination},
+			}}, 4, 1)
+			handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
+				"REPORTS_TOKEN": "reports-secret",
+			}))
+			if err != nil {
+				t.Fatalf("newUploadHTTPHandler() error = %v", err)
+			}
+			server := httptest.NewServer(handler)
+			defer server.Close()
+
+			runID := submitHTTPUpload(t, server, "reports-secret", tt.contentType, bundleArchive(t, tt.compressed, testutil.BundleOptions{}))
+			record := waitForHTTPUploadStatus(t, server, runID, UploadStatusSucceeded)
+
+			if record.Report == nil {
+				t.Fatal("completed status report = nil, want run report")
+			}
+			if record.Report.Summary.Status != "ok" {
+				t.Fatalf("summary status = %q, want ok", record.Report.Summary.Status)
+			}
+			if got, want := len(record.Report.Actions), 2; got != want {
+				t.Fatalf("action count = %d, want %d", got, want)
+			}
+			assertPublishedBundle(t, firstDestination)
+			assertPublishedBundle(t, secondDestination)
+		})
+	}
+}
+
+func TestHTTPUploadInvalidArchiveIsRejectedWithoutRunID(t *testing.T) {
+	destination := t.TempDir()
+	coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+		id:           "reports",
+		tokenEnv:     "REPORTS_TOKEN",
+		stagingPath:  filepath.Join(t.TempDir(), "reports"),
+		destinations: []string{destination},
+	}}, 4, 1))
+	handler := uploadHTTPHandler{
+		coordinator: coordinator,
+		tokens:      map[string]string{"reports-secret": "reports"},
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	status, body := postHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("not a tar archive"))
+	if status != http.StatusBadRequest {
+		t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusBadRequest, body)
+	}
+	if strings.Contains(body, "run_id") || strings.Contains(body, "reports-secret") {
+		t.Fatalf("invalid archive response exposed run id or token: %s", body)
+	}
+	if got := coordinator.QueueDepth(); got != 0 {
+		t.Fatalf("queue depth = %d, want 0", got)
+	}
+	assertDirectoryEmpty(t, destination)
+}
+
+func TestHTTPUploadIdempotencyReturnsOriginalRunForSameBundle(t *testing.T) {
+	destination := t.TempDir()
+	coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+		id:           "reports",
+		tokenEnv:     "REPORTS_TOKEN",
+		stagingPath:  filepath.Join(t.TempDir(), "reports"),
+		destinations: []string{destination},
+	}}, 4, 1))
+	handler := uploadHTTPHandler{
+		coordinator: coordinator,
+		tokens:      map[string]string{"reports-secret": "reports"},
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	firstRunID := submitHTTPUploadWithKey(t, server, "reports-secret", ingest.ContentTypeTar, "same-key", bundleArchive(t, false, testutil.BundleOptions{}))
+	waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
+
+	secondRunID := submitHTTPUploadWithKey(t, server, "reports-secret", ingest.ContentTypeGzip, "same-key", bundleArchive(t, true, testutil.BundleOptions{}))
+	if secondRunID != firstRunID {
+		t.Fatalf("second run id = %q, want original %q", secondRunID, firstRunID)
+	}
+	if got := coordinator.QueueDepth(); got != 0 {
+		t.Fatalf("queue depth = %d, want no duplicate run queued", got)
+	}
+}
+
+func TestHTTPUploadIdempotencyReturnsConflictForDifferentBundle(t *testing.T) {
+	destination := t.TempDir()
+	coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+		id:           "reports",
+		tokenEnv:     "REPORTS_TOKEN",
+		stagingPath:  filepath.Join(t.TempDir(), "reports"),
+		destinations: []string{destination},
+	}}, 4, 1))
+	handler := uploadHTTPHandler{
+		coordinator: coordinator,
+		tokens:      map[string]string{"reports-secret": "reports"},
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	firstRunID := submitHTTPUploadWithKey(t, server, "reports-secret", ingest.ContentTypeTar, "same-key", bundleArchive(t, false, testutil.BundleOptions{}))
+	waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
+
+	status, body := postHTTPUploadWithKey(t, server, "reports-secret", ingest.ContentTypeTar, "same-key", bundleArchive(t, false, testutil.BundleOptions{
+		ID: "weather.daily.brentwood.2026-05-31",
+	}))
+	if status != http.StatusConflict {
+		t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusConflict, body)
+	}
+	if strings.Contains(body, "reports-secret") {
+		t.Fatalf("conflict response exposed token: %s", body)
+	}
+}
+
+func TestHTTPUploadOversizedArchiveIsRejectedWithoutRunID(t *testing.T) {
+	destination := t.TempDir()
+	stagingPath := filepath.Join(t.TempDir(), "reports")
+	cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+		id:           "reports",
+		tokenEnv:     "REPORTS_TOKEN",
+		stagingPath:  stagingPath,
+		destinations: []string{destination},
+	}}, 4, 1)
+	size := config.ByteSize(4)
+	cfg.Server.HTTP.MaxUploadSize = &size
+	cfg.Pipelines[0].Source.Upload.MaxUploadSize = &size
+	coordinator := NewUploadCoordinator(context.Background(), cfg)
+	handler := uploadHTTPHandler{
+		coordinator: coordinator,
+		tokens:      map[string]string{"reports-secret": "reports"},
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	status, body := postHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{}))
+	if status != http.StatusRequestEntityTooLarge {
+		t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusRequestEntityTooLarge, body)
+	}
+	if strings.Contains(body, "run_id") || strings.Contains(body, "reports-secret") {
+		t.Fatalf("oversized response exposed run id or token: %s", body)
+	}
+	if got := coordinator.QueueDepth(); got != 0 {
+		t.Fatalf("queue depth = %d, want 0", got)
+	}
+	assertDirectoryEmpty(t, stagingPath)
+	assertDirectoryEmpty(t, destination)
+}
+
+func TestHTTPUploadSamePipelineRequestsSerialize(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	started := make(chan struct{}, 1)
+	coordinator := newUploadCoordinator(ctx, httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+		id:           "reports",
+		tokenEnv:     "REPORTS_TOKEN",
+		stagingPath:  filepath.Join(t.TempDir(), "reports"),
+		destinations: []string{t.TempDir()},
+	}}, 4, 2), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			select {
+			case started <- struct{}{}:
+			default:
+			}
+			<-release
+			return RunReport{}, nil
+		},
+	})
+	handler := uploadHTTPHandler{
+		coordinator: coordinator,
+		tokens:      map[string]string{"reports-secret": "reports"},
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	firstRunID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("first"))
+	waitForRunStart(t, started)
+	first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusRunning)
+	secondRunID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("second"))
+	second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusQueued)
+
+	if first.PipelineID != "reports" || second.PipelineID != "reports" {
+		t.Fatalf("statuses = %#v %#v, want same pipeline", first, second)
+	}
+	if got := coordinator.RunningCount(); got != 1 {
+		t.Fatalf("running count = %d, want 1", got)
+	}
+	close(release)
+	waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
+	waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
+}
+
+func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	started := make(chan string, 2)
+	coordinator := newUploadCoordinator(ctx, httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
+		{
+			id:           "reports-one",
+			tokenEnv:     "REPORTS_ONE_TOKEN",
+			stagingPath:  filepath.Join(t.TempDir(), "reports-one"),
+			destinations: []string{t.TempDir()},
+		},
+		{
+			id:           "reports-two",
+			tokenEnv:     "REPORTS_TWO_TOKEN",
+			stagingPath:  filepath.Join(t.TempDir(), "reports-two"),
+			destinations: []string{t.TempDir()},
+		},
+	}, 4, 2), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			started <- options.PipelineID
+			<-release
+			return RunReport{}, nil
+		},
+	})
+	handler := uploadHTTPHandler{
+		coordinator: coordinator,
+		tokens: map[string]string{
+			"one-secret": "reports-one",
+			"two-secret": "reports-two",
+		},
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	firstRunID := submitHTTPUpload(t, server, "one-secret", ingest.ContentTypeTar, []byte("first"))
+	secondRunID := submitHTTPUpload(t, server, "two-secret", ingest.ContentTypeTar, []byte("second"))
+	waitForStartedPipelines(t, started, "reports-one", "reports-two")
+	waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusRunning)
+	waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusRunning)
+
+	if got := coordinator.RunningCount(); got != 2 {
+		t.Fatalf("running count = %d, want 2", got)
+	}
+	close(release)
+	waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
+	waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
+}
+
+type httpUploadPipelineSpec struct {
+	id           string
+	tokenEnv     string
+	stagingPath  string
+	destinations []string
+}
+
+func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpec, queueSize, maxConcurrency int) config.Config {
+	t.Helper()
+	size := config.ByteSize(1024 * 1024)
+	retention := config.Duration(time.Minute)
+	cfg := config.Config{
+		Server: config.Server{HTTP: config.HTTPServer{
+			Bind:           config.DefaultHTTPBind,
+			StagingRoot:    t.TempDir(),
+			MaxUploadSize:  &size,
+			QueueSize:      queueSize,
+			MaxConcurrency: maxConcurrency,
+			Retention:      &retention,
+		}},
+	}
+	for _, spec := range pipelines {
+		pipeline := config.Pipeline{
+			ID: spec.id,
+			Source: config.Backend{
+				Backend: config.BackendHTTPUpload,
+				Upload: config.HTTPUpload{
+					TokenEnv:      spec.tokenEnv,
+					StagingPath:   spec.stagingPath,
+					MaxUploadSize: &size,
+				},
+			},
+		}
+		for index, destination := range spec.destinations {
+			pipeline.Destinations = append(pipeline.Destinations, config.Destination{
+				ID:      fmt.Sprintf("archive-%d", index+1),
+				Backend: config.BackendLocal,
+				Path:    destination,
+				Publish: &config.PublishPolicy{Source: true},
+			})
+		}
+		cfg.Pipelines = append(cfg.Pipelines, pipeline)
+	}
+	config.ApplyDefaults(&cfg)
+	return cfg
+}
+
+func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) UploadRunID {
+	t.Helper()
+	status, responseBody := postHTTPUpload(t, server, token, contentType, body)
+	return decodeAcceptedHTTPUpload(t, status, responseBody)
+}
+
+func submitHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, contentType, key string, body []byte) UploadRunID {
+	t.Helper()
+	status, responseBody := postHTTPUploadWithKey(t, server, token, contentType, key, body)
+	return decodeAcceptedHTTPUpload(t, status, responseBody)
+}
+
+func decodeAcceptedHTTPUpload(t *testing.T, status int, responseBody string) UploadRunID {
+	t.Helper()
+	if status != http.StatusAccepted {
+		t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusAccepted, responseBody)
+	}
+	var accepted uploadAcceptedResponse
+	if err := json.Unmarshal([]byte(responseBody), &accepted); err != nil {
+		t.Fatalf("decode accepted response: %v", err)
+	}
+	if accepted.RunID == "" || accepted.Status != UploadStatusAccepted {
+		t.Fatalf("accepted response = %#v, want run id and accepted status", accepted)
+	}
+	return accepted.RunID
+}
+
+func postHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) (int, string) {
+	t.Helper()
+	return postHTTPUploadWithKey(t, server, token, contentType, "", body)
+}
+
+func postHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, contentType, key string, body []byte) (int, string) {
+	t.Helper()
+	request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("NewRequest() error = %v", err)
+	}
+	request.Header.Set("Authorization", "Bearer "+token)
+	request.Header.Set("Content-Type", contentType)
+	if key != "" {
+		request.Header.Set("Idempotency-Key", key)
+	}
+	response, err := server.Client().Do(request)
+	if err != nil {
+		t.Fatalf("POST /upload error = %v", err)
+	}
+	defer response.Body.Close()
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	return response.StatusCode, string(data)
+}
+
+func waitForHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID, status UploadStatus) UploadRunRecord {
+	t.Helper()
+	deadline := time.Now().Add(3 * time.Second)
+	var latest UploadRunRecord
+	var latestStatus int
+	for time.Now().Before(deadline) {
+		latest, latestStatus = getHTTPUploadStatus(t, server, runID)
+		if latestStatus == http.StatusOK && latest.Status == status {
+			return latest
+		}
+		time.Sleep(time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for status %s; latest HTTP status=%d record=%#v", status, latestStatus, latest)
+	return UploadRunRecord{}
+}
+
+func getHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID) (UploadRunRecord, int) {
+	t.Helper()
+	response, err := server.Client().Get(server.URL + "/runs/" + string(runID))
+	if err != nil {
+		t.Fatalf("GET /runs error = %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return UploadRunRecord{}, response.StatusCode
+	}
+	var record UploadRunRecord
+	if err := json.NewDecoder(response.Body).Decode(&record); err != nil {
+		t.Fatalf("decode run status: %v", err)
+	}
+	return record, response.StatusCode
+}
+
+func bundleArchive(t *testing.T, compressed bool, opts testutil.BundleOptions) []byte {
+	t.Helper()
+	root := t.TempDir()
+	testutil.WriteSourceBundle(t, root, "", opts)
+	return tarDirectory(t, root, compressed)
+}
+
+func tarDirectory(t *testing.T, root string, compressed bool) []byte {
+	t.Helper()
+	var output bytes.Buffer
+	var writer io.WriteCloser = nopWriteCloser{writer: &output}
+	if compressed {
+		gzipWriter := gzip.NewWriter(&output)
+		writer = gzipWriter
+	}
+	tarWriter := tar.NewWriter(writer)
+	if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		relative, err := filepath.Rel(root, filePath)
+		if err != nil {
+			return err
+		}
+		data, err := os.ReadFile(filePath)
+		if err != nil {
+			return err
+		}
+		header := &tar.Header{
+			Name: filepath.ToSlash(relative),
+			Mode: 0o600,
+			Size: int64(len(data)),
+		}
+		if err := tarWriter.WriteHeader(header); err != nil {
+			return err
+		}
+		if _, err := tarWriter.Write(data); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("walk bundle: %v", err)
+	}
+	if err := tarWriter.Close(); err != nil {
+		t.Fatalf("close tar: %v", err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatalf("close archive: %v", err)
+	}
+	return output.Bytes()
+}
+
+type nopWriteCloser struct {
+	writer io.Writer
+}
+
+func (writer nopWriteCloser) Write(data []byte) (int, error) {
+	return writer.writer.Write(data)
+}
+
+func (writer nopWriteCloser) Close() error {
+	return nil
+}
+
+func assertPublishedBundle(t *testing.T, destinationRoot string) {
+	t.Helper()
+	testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
+	testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
+	if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
+		t.Fatalf("destination state stat: %v", err)
+	}
+}
+
+func assertDirectoryEmpty(t *testing.T, root string) {
+	t.Helper()
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		t.Fatalf("ReadDir() error = %v", err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("directory %s has %d entries, want empty", root, len(entries))
+	}
+}
+
+func waitForRunStart(t *testing.T, started <-chan struct{}) {
+	t.Helper()
+	select {
+	case <-started:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for run start")
+	}
+}
+
+func waitForStartedPipelines(t *testing.T, started <-chan string, want ...string) {
+	t.Helper()
+	remaining := map[string]bool{}
+	for _, pipelineID := range want {
+		remaining[pipelineID] = true
+	}
+	deadline := time.After(time.Second)
+	for len(remaining) > 0 {
+		select {
+		case pipelineID := <-started:
+			delete(remaining, pipelineID)
+		case <-deadline:
+			t.Fatalf("timed out waiting for pipelines to start; remaining=%v", remaining)
+		}
+	}
+}

internal/app/upload_http_test.go

@@ -0,0 +1,386 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+)
+
+type fakeUploadCoordinator struct {
+	canAccept bool
+	submit    func(context.Context, UploadRequest) (UploadRunRecord, error)
+	status    func(UploadRunID) (UploadRunRecord, bool)
+}
+
+func (fake fakeUploadCoordinator) CanAccept() bool {
+	return fake.canAccept
+}
+
+func (fake fakeUploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) {
+	if fake.submit == nil {
+		return UploadRunRecord{}, errors.New("unexpected submit")
+	}
+	return fake.submit(ctx, request)
+}
+
+func (fake fakeUploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
+	if fake.status == nil {
+		return UploadRunRecord{}, false
+	}
+	return fake.status(runID)
+}
+
+func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) {
+	cfg := uploadHTTPTestConfig()
+
+	_, err := resolveUploadTokens(cfg, config.NewEnvironment(nil, func(string) (string, bool) {
+		return "", false
+	}))
+	if err == nil || !strings.Contains(err.Error(), "UPLOAD_TOKEN") {
+		t.Fatalf("resolveUploadTokens() error = %v, want missing UPLOAD_TOKEN", err)
+	}
+
+	cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
+		ID: "weekly",
+		Source: config.Backend{
+			Backend: config.BackendHTTPUpload,
+			Upload:  config.HTTPUpload{TokenEnv: "OTHER_UPLOAD_TOKEN"},
+		},
+		Destinations: cfg.Pipelines[0].Destinations,
+	})
+	config.ApplyDefaults(&cfg)
+	secret := "super-secret-token"
+	_, err = resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{
+		"UPLOAD_TOKEN":       secret,
+		"OTHER_UPLOAD_TOKEN": secret,
+	}))
+	if err == nil {
+		t.Fatal("resolveUploadTokens() error = nil, want duplicate token error")
+	}
+	if strings.Contains(err.Error(), secret) {
+		t.Fatalf("duplicate token error exposed secret value: %q", err)
+	}
+}
+
+func TestNewUploadHTTPHandlerAcceptsDefaultedConfig(t *testing.T) {
+	cfg := uploadHTTPTestConfig()
+	cfg.Server.HTTP.Bind = ""
+	cfg.Server.HTTP.StagingRoot = ""
+	cfg.Server.HTTP.MaxUploadSize = nil
+	cfg.Server.HTTP.QueueSize = 0
+	cfg.Server.HTTP.MaxConcurrency = 0
+	cfg.Server.HTTP.Retention = nil
+	cfg.Pipelines[0].Source.Upload.StagingPath = ""
+	cfg.Pipelines[0].Source.Upload.MaxUploadSize = nil
+
+	handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
+		"UPLOAD_TOKEN": "secret",
+	}))
+	if err != nil {
+		t.Fatalf("newUploadHTTPHandler() error = %v", err)
+	}
+	if handler == nil {
+		t.Fatal("newUploadHTTPHandler() = nil")
+	}
+}
+
+func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
+	var submitted UploadRequest
+	handler := uploadHTTPHandler{
+		coordinator: fakeUploadCoordinator{
+			submit: func(_ context.Context, request UploadRequest) (UploadRunRecord, error) {
+				submitted = request
+				body, err := io.ReadAll(request.Body)
+				if err != nil {
+					t.Fatalf("read submitted body: %v", err)
+				}
+				if string(body) != "archive" {
+					t.Fatalf("submitted body = %q, want archive", body)
+				}
+				return UploadRunRecord{ID: "reports.20260603T120000Z.abcdef12", Status: UploadStatusAccepted}, nil
+			},
+		},
+		tokens: map[string]string{"valid-token": "reports"},
+	}
+	recorder := httptest.NewRecorder()
+	request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
+	request.Header.Set("Authorization", "Bearer valid-token")
+	request.Header.Set("Content-Type", "application/x-tar")
+	request.Header.Set("Idempotency-Key", "producer.retry:20260603")
+
+	handler.ServeHTTP(recorder, request)
+
+	if recorder.Code != http.StatusAccepted {
+		t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusAccepted, recorder.Body.String())
+	}
+	if submitted.PipelineID != "reports" {
+		t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID)
+	}
+	if submitted.IdempotencyKey != "producer.retry:20260603" {
+		t.Fatalf("submitted idempotency key = %q, want producer.retry:20260603", submitted.IdempotencyKey)
+	}
+	var response uploadAcceptedResponse
+	if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+	if response.RunID != "reports.20260603T120000Z.abcdef12" || response.Status != UploadStatusAccepted {
+		t.Fatalf("response = %#v, want accepted run id", response)
+	}
+	if strings.Contains(recorder.Body.String(), "valid-token") {
+		t.Fatalf("response exposed token: %q", recorder.Body.String())
+	}
+}
+
+func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
+	handler := uploadHTTPHandler{
+		coordinator: fakeUploadCoordinator{},
+		tokens:      map[string]string{"valid-token": "reports"},
+	}
+
+	for _, authHeader := range []string{"", "Bearer wrong-token"} {
+		recorder := httptest.NewRecorder()
+		request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
+		request.Header.Set("Authorization", authHeader)
+		request.Header.Set("Content-Type", "application/x-tar")
+
+		handler.ServeHTTP(recorder, request)
+
+		if recorder.Code != http.StatusUnauthorized {
+			t.Fatalf("auth %q status = %d, want %d", authHeader, recorder.Code, http.StatusUnauthorized)
+		}
+		if strings.Contains(recorder.Body.String(), "valid-token") || strings.Contains(recorder.Body.String(), "wrong-token") {
+			t.Fatalf("unauthorized response exposed token: %q", recorder.Body.String())
+		}
+	}
+}
+
+func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t *testing.T) {
+	tests := []struct {
+		name        string
+		url         string
+		contentType string
+		keyValues   []string
+		body        io.Reader
+		wantStatus  int
+	}{
+		{
+			name:        "unsupported content type",
+			url:         "/upload",
+			contentType: "application/zip",
+			body:        strings.NewReader("archive"),
+			wantStatus:  http.StatusUnsupportedMediaType,
+		},
+		{
+			name:        "invalid key syntax",
+			url:         "/upload",
+			contentType: "application/x-tar",
+			keyValues:   []string{"bad key"},
+			body:        strings.NewReader("archive"),
+			wantStatus:  http.StatusBadRequest,
+		},
+		{
+			name:        "empty key",
+			url:         "/upload",
+			contentType: "application/x-tar",
+			keyValues:   []string{""},
+			body:        strings.NewReader("archive"),
+			wantStatus:  http.StatusBadRequest,
+		},
+		{
+			name:        "too long key",
+			url:         "/upload",
+			contentType: "application/x-tar",
+			keyValues:   []string{strings.Repeat("a", 129)},
+			body:        strings.NewReader("archive"),
+			wantStatus:  http.StatusBadRequest,
+		},
+		{
+			name:        "multiple keys",
+			url:         "/upload",
+			contentType: "application/x-tar",
+			keyValues:   []string{"one", "two"},
+			body:        strings.NewReader("archive"),
+			wantStatus:  http.StatusBadRequest,
+		},
+		{
+			name:        "submitted pipeline id",
+			url:         "/upload?pipeline_id=reports",
+			contentType: "application/x-tar",
+			body:        strings.NewReader("archive"),
+			wantStatus:  http.StatusBadRequest,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			handler := uploadHTTPHandler{
+				coordinator: fakeUploadCoordinator{
+					submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
+						t.Fatal("Submit should not be called")
+						return UploadRunRecord{}, nil
+					},
+				},
+				tokens: map[string]string{"valid-token": "reports"},
+			}
+			recorder := httptest.NewRecorder()
+			request := httptest.NewRequest(http.MethodPost, tt.url, tt.body)
+			request.Header.Set("Authorization", "Bearer valid-token")
+			request.Header.Set("Content-Type", tt.contentType)
+			for _, value := range tt.keyValues {
+				request.Header.Add("Idempotency-Key", value)
+			}
+
+			handler.ServeHTTP(recorder, request)
+
+			if recorder.Code != tt.wantStatus {
+				t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
+			}
+		})
+	}
+}
+
+func TestUploadHTTPHandlerMapsSubmitErrors(t *testing.T) {
+	tests := []struct {
+		name       string
+		err        error
+		wantStatus int
+		wantBody   string
+	}{
+		{name: "oversized", err: ingest.ErrUploadTooLarge, wantStatus: http.StatusRequestEntityTooLarge},
+		{name: "unsupported", err: ingest.ErrUnsupportedContentType, wantStatus: http.StatusUnsupportedMediaType},
+		{name: "full queue", err: UploadQueueFullError{QueueSize: 1}, wantStatus: http.StatusServiceUnavailable},
+		{name: "idempotency conflict", err: UploadIdempotencyConflictError{}, wantStatus: http.StatusConflict, wantBody: "different source manifest"},
+		{name: "idempotency in progress", err: UploadIdempotencyConflictError{Retryable: true}, wantStatus: http.StatusConflict, wantBody: `"retryable":true`},
+		{name: "malformed", err: errors.New("malformed archive"), wantStatus: http.StatusBadRequest},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			handler := uploadHTTPHandler{
+				coordinator: fakeUploadCoordinator{
+					canAccept: true,
+					submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
+						return UploadRunRecord{}, tt.err
+					},
+				},
+				tokens: map[string]string{"valid-token": "reports"},
+			}
+			recorder := httptest.NewRecorder()
+			request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
+			request.Header.Set("Authorization", "Bearer valid-token")
+			request.Header.Set("Content-Type", "application/x-tar")
+
+			handler.ServeHTTP(recorder, request)
+
+			if recorder.Code != tt.wantStatus {
+				t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
+			}
+			if tt.wantBody != "" && !strings.Contains(recorder.Body.String(), tt.wantBody) {
+				t.Fatalf("body = %q, want substring %q", recorder.Body.String(), tt.wantBody)
+			}
+		})
+	}
+}
+
+func TestUploadHTTPHandlerRunStatusAndHealth(t *testing.T) {
+	finishedAt := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
+	handler := uploadHTTPHandler{
+		coordinator: fakeUploadCoordinator{
+			canAccept: true,
+			status: func(runID UploadRunID) (UploadRunRecord, bool) {
+				if runID != "reports.20260603T120000Z.abcdef12" {
+					return UploadRunRecord{}, false
+				}
+				return UploadRunRecord{
+					ID:         runID,
+					PipelineID: "reports",
+					Status:     UploadStatusSucceeded,
+					FinishedAt: &finishedAt,
+				}, true
+			},
+		},
+		tokens: map[string]string{"valid-token": "reports"},
+	}
+
+	recorder := httptest.NewRecorder()
+	handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+	if recorder.Code != http.StatusOK {
+		t.Fatalf("health status = %d, want %d", recorder.Code, http.StatusOK)
+	}
+
+	recorder = httptest.NewRecorder()
+	handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/reports.20260603T120000Z.abcdef12", nil))
+	if recorder.Code != http.StatusOK {
+		t.Fatalf("run status = %d, want %d; body = %q", recorder.Code, http.StatusOK, recorder.Body.String())
+	}
+	var record UploadRunRecord
+	if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
+		t.Fatalf("decode run status: %v", err)
+	}
+	if record.ID != "reports.20260603T120000Z.abcdef12" || record.Status != UploadStatusSucceeded {
+		t.Fatalf("record = %#v, want succeeded run status", record)
+	}
+
+	recorder = httptest.NewRecorder()
+	handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/unknown", nil))
+	if recorder.Code != http.StatusNotFound {
+		t.Fatalf("unknown run status = %d, want %d", recorder.Code, http.StatusNotFound)
+	}
+}
+
+type countingReader struct {
+	reader io.Reader
+	reads  int
+}
+
+func (reader *countingReader) Read(data []byte) (int, error) {
+	reader.reads++
+	return reader.reader.Read(data)
+}
+
+func uploadHTTPTestConfig() config.Config {
+	size := config.ByteSize(1024)
+	retention := config.Duration(24 * time.Hour)
+	cfg := config.Config{
+		Server: config.Server{HTTP: config.HTTPServer{
+			Bind:           config.DefaultHTTPBind,
+			StagingRoot:    "/tmp/distributor-test",
+			MaxUploadSize:  &size,
+			QueueSize:      2,
+			MaxConcurrency: 1,
+			Retention:      &retention,
+		}},
+		Pipelines: []config.Pipeline{{
+			ID: "reports",
+			Source: config.Backend{
+				Backend: config.BackendHTTPUpload,
+				Upload: config.HTTPUpload{
+					TokenEnv:      "UPLOAD_TOKEN",
+					StagingPath:   "/tmp/distributor-test/reports",
+					MaxUploadSize: &size,
+				},
+			},
+			Destinations: []config.Destination{{
+				ID:      "local",
+				Backend: config.BackendLocal,
+				Path:    "/tmp/distributor-output",
+				Publish: &config.PublishPolicy{Source: true},
+			}},
+		}},
+	}
+	config.ApplyDefaults(&cfg)
+	return cfg
+}
+
+func uploadHTTPTestEnvironment(values map[string]string) config.Environment {
+	return config.NewEnvironment(values, func(string) (string, bool) {
+		return "", false
+	})
+}

internal/app/validate.go

@@ -5,28 +5,84 @@ import (
 	"fmt"
 	"io"
 
-	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
 )
 
 type ValidateOptions struct {
-	Path   string
-	Stdout io.Writer
+	Path         string
+	ConfigPath   string
+	PipelineID   string
+	BundlePath   string
+	Stdout       io.Writer
+	OutputFormat OutputFormat
 }
 
 func Validate(ctx context.Context, options ValidateOptions) error {
-	if options.Path == "" {
-		return fmt.Errorf("validate command requires a path")
+	return validateWithBackendFactory(ctx, options, newBackendFactoryWithEnvironment)
+}
+
+func validateWithBackendFactory(ctx context.Context, options ValidateOptions, provider backendFactoryProvider) error {
+	if err := ValidateOutputFormat(options.OutputFormat); err != nil {
+		return err
 	}
-	backend, err := newBackendFactory().openLocalPath(ctx, options.Path)
+	selection, err := selectSourceBundles(ctx, sourceCommandOptions{
+		CommandName: "validate",
+		Path:        options.Path,
+		ConfigPath:  options.ConfigPath,
+		PipelineID:  options.PipelineID,
+		BundlePath:  options.BundlePath,
+	}, provider)
 	if err != nil {
 		return err
 	}
-	bundles, err := bundle.Discover(ctx, backend, "")
+	return writeValidateResult(options, selection)
+}
+
+func validateConfigWithBackendFactory(ctx context.Context, cfg config.Config, options ValidateOptions, provider backendFactoryProvider) error {
+	if err := ValidateOutputFormat(options.OutputFormat); err != nil {
+		return err
+	}
+	selection, err := selectSourceBundlesFromConfig(ctx, cfg, sourceCommandOptions{
+		CommandName: "validate",
+		PipelineID:  options.PipelineID,
+		BundlePath:  options.BundlePath,
+	}, provider)
 	if err != nil {
 		return err
 	}
+	return writeValidateResult(options, selection)
+}
+
+func writeValidateResult(options ValidateOptions, selection sourceSelection) error {
+	if IsJSONOutput(options.OutputFormat) {
+		return WriteJSONEnvelope(options.Stdout, "validate", true, selection.Warnings, validateResultFromSelection(selection), nil)
+	}
+	var err error
 	if options.Stdout != nil {
-		_, err = fmt.Fprintf(options.Stdout, "Validated %d bundle(s)\n", len(bundles))
+		if err := writeWarnings(options.Stdout, selection.Warnings); err != nil {
+			return err
+		}
+		if selection.ConfigMode {
+			_, err = fmt.Fprintf(options.Stdout, "Validated %d bundle(s) for pipeline %s source %s\n", len(selection.Bundles), selection.PipelineID, selection.SourceBackend)
+		} else {
+			_, err = fmt.Fprintf(options.Stdout, "Validated %d bundle(s)\n", len(selection.Bundles))
+		}
 	}
 	return err
 }
+
+type validateResult struct {
+	PipelineID    string                `json:"pipeline_id,omitempty"`
+	SourceBackend string                `json:"source_backend,omitempty"`
+	BundleCount   int                   `json:"bundle_count"`
+	Bundles       []bundleSummaryResult `json:"bundles"`
+}
+
+func validateResultFromSelection(selection sourceSelection) validateResult {
+	return validateResult{
+		PipelineID:    selection.PipelineID,
+		SourceBackend: selection.SourceBackend,
+		BundleCount:   len(selection.Bundles),
+		Bundles:       bundleSummariesFromBundles(selection.Bundles),
+	}
+}

internal/app/validate_test.go

@@ -3,9 +3,16 @@ package app
 import (
 	"bytes"
 	"context"
+	"encoding/json"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
+	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
 )
 
 func TestValidateLocalBundle(t *testing.T) {
@@ -31,9 +38,227 @@ func TestValidateExampleSourceBundle(t *testing.T) {
 	}
 }
 
+func TestValidateConfiguredLocalSource(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	var stdout bytes.Buffer
+
+	err := Validate(context.Background(), ValidateOptions{
+		ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
+		PipelineID: "reports",
+		Stdout:     &stdout,
+	})
+
+	if err != nil {
+		t.Fatalf("Validate() configured source error = %v", err)
+	}
+	if got, want := stdout.String(), "Validated 1 bundle(s) for pipeline reports source local\n"; got != want {
+		t.Fatalf("stdout = %q, want %q", got, want)
+	}
+}
+
+func TestValidateConfiguredSourceBundlePath(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "daily/one", testutil.BundleOptions{ID: "reports.one"})
+	testutil.WriteSourceBundle(t, sourceRoot, "daily/two", testutil.BundleOptions{ID: "reports.two"})
+	var stdout bytes.Buffer
+
+	err := Validate(context.Background(), ValidateOptions{
+		ConfigPath:   testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
+		PipelineID:   "reports",
+		BundlePath:   "daily/two",
+		Stdout:       &stdout,
+		OutputFormat: OutputFormatJSON,
+	})
+
+	if err != nil {
+		t.Fatalf("Validate() configured bundle error = %v", err)
+	}
+	result := decodeAppResult(t, stdout.String())
+	if result["pipeline_id"] != "reports" || result["source_backend"] != "local" || result["bundle_count"] != float64(1) {
+		t.Fatalf("result = %#v, want configured source summary", result)
+	}
+	bundles, ok := result["bundles"].([]any)
+	if !ok || len(bundles) != 1 {
+		t.Fatalf("bundles = %#v, want one bundle", result["bundles"])
+	}
+	sourceBundle, ok := bundles[0].(map[string]any)
+	if !ok || sourceBundle["path"] != "daily/two" || sourceBundle["id"] != "reports.two" {
+		t.Fatalf("bundle = %#v, want narrowed bundle", sourceBundle)
+	}
+}
+
+func TestValidateConfiguredRemoteSourcesThroughStorageAbstraction(t *testing.T) {
+	s3Source := fake.New()
+	testutil.WriteFakeSourceBundle(t, s3Source, "", testutil.BundleOptions{ID: "reports.s3"})
+	sshSource := fake.New()
+	testutil.WriteFakeSourceBundle(t, sshSource, "daily", testutil.BundleOptions{ID: "reports.ssh"})
+	cfg := config.Config{Pipelines: []config.Pipeline{
+		{
+			ID: "s3-reports",
+			Source: config.Backend{
+				Backend:  config.BackendS3,
+				Endpoint: "http://s3.test",
+				Bucket:   "source-bucket",
+			},
+			Destinations: []config.Destination{{
+				ID:      "archive",
+				Backend: config.BackendLocal,
+				Path:    t.TempDir(),
+			}},
+		},
+		{
+			ID: "ssh-reports",
+			Source: config.Backend{
+				Backend: config.BackendSSH,
+				Host:    "ssh.test",
+				Path:    "/source",
+			},
+			Destinations: []config.Destination{{
+				ID:      "archive",
+				Backend: config.BackendLocal,
+				Path:    t.TempDir(),
+			}},
+		},
+	}}
+	config.ApplyDefaults(&cfg)
+	provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
+		"s3:source-bucket": s3Source,
+		"ssh:/source":      sshSource,
+	})
+
+	var s3Stdout bytes.Buffer
+	if err := validateConfigWithBackendFactory(context.Background(), cfg, ValidateOptions{
+		PipelineID:   "s3-reports",
+		Stdout:       &s3Stdout,
+		OutputFormat: OutputFormatJSON,
+	}, provider); err != nil {
+		t.Fatalf("validate s3 source error = %v", err)
+	}
+	s3Result := decodeAppResult(t, s3Stdout.String())
+	if s3Result["source_backend"] != "s3" || s3Result["bundle_count"] != float64(1) {
+		t.Fatalf("s3 result = %#v, want one s3 bundle", s3Result)
+	}
+
+	var sshStdout bytes.Buffer
+	if err := validateConfigWithBackendFactory(context.Background(), cfg, ValidateOptions{
+		PipelineID: "ssh-reports",
+		BundlePath: "daily",
+		Stdout:     &sshStdout,
+	}, provider); err != nil {
+		t.Fatalf("validate ssh source error = %v", err)
+	}
+	if !strings.Contains(sshStdout.String(), "pipeline ssh-reports source ssh") {
+		t.Fatalf("ssh stdout = %q, want ssh source summary", sshStdout.String())
+	}
+}
+
+func TestValidateConfiguredSourceLoadsSecretsBeforeOpeningBackend(t *testing.T) {
+	sourceRoot := filepath.Join(t.TempDir(), "missing-source")
+	destinationRoot := t.TempDir()
+	configPath := writeConfigFile(t, `
+secrets:
+  directory: `+filepath.Join(t.TempDir(), "missing-secrets")+`
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: archive
+        backend: local
+        path: `+destinationRoot+`
+`)
+
+	err := Validate(context.Background(), ValidateOptions{ConfigPath: configPath, PipelineID: "reports"})
+	if err == nil {
+		t.Fatal("Validate() error = nil, want secrets directory error")
+	}
+	if !strings.Contains(err.Error(), "load secrets directory") {
+		t.Fatalf("Validate() error = %v, want secrets directory error", err)
+	}
+	if strings.Contains(err.Error(), "missing-source") {
+		t.Fatalf("Validate() error = %v, opened source before loading secrets", err)
+	}
+}
+
+func TestValidateConfiguredSourcePrintsSecretConflictWarningWithoutValues(t *testing.T) {
+	name := "DISTRIBUTOR_TEST_VALIDATE_SECRET"
+	t.Setenv(name, "process-value")
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	secretsRoot := t.TempDir()
+	if err := os.WriteFile(filepath.Join(secretsRoot, name), []byte("secret-value\n"), 0o600); err != nil {
+		t.Fatalf("write secret: %v", err)
+	}
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	configPath := writeConfigFile(t, `
+secrets:
+  directory: `+secretsRoot+`
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: archive
+        backend: local
+        path: `+destinationRoot+`
+`)
+
+	var stdout bytes.Buffer
+	err := Validate(context.Background(), ValidateOptions{
+		ConfigPath: configPath,
+		PipelineID: "reports",
+		Stdout:     &stdout,
+	})
+	if err != nil {
+		t.Fatalf("Validate() error = %v", err)
+	}
+	output := stdout.String()
+	if !strings.Contains(output, "secret "+name+" ignored because the real environment already has that variable") {
+		t.Fatalf("stdout = %q, want secret conflict warning", output)
+	}
+	if !strings.Contains(output, "Validated 1 bundle(s) for pipeline reports source local") {
+		t.Fatalf("stdout = %q, want validate summary", output)
+	}
+	if strings.Contains(output, "process-value") || strings.Contains(output, "secret-value") {
+		t.Fatalf("stdout exposed secret values: %q", output)
+	}
+}
+
+func TestValidateConfiguredSourceRequiresPipeline(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+
+	err := Validate(context.Background(), ValidateOptions{
+		ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
+	})
+
+	if err == nil || !strings.Contains(err.Error(), "requires --pipeline") {
+		t.Fatalf("Validate() error = %v, want required pipeline", err)
+	}
+}
+
 func TestValidateRequiresPath(t *testing.T) {
 	err := Validate(context.Background(), ValidateOptions{})
 	if err == nil || !strings.Contains(err.Error(), "requires a path") {
 		t.Fatalf("Validate() error = %v, want required path", err)
 	}
 }
+
+func decodeAppResult(t *testing.T, output string) map[string]any {
+	t.Helper()
+	var envelope map[string]any
+	if err := json.Unmarshal([]byte(output), &envelope); err != nil {
+		t.Fatalf("decode output: %v; output = %q", err, output)
+	}
+	result, ok := envelope["result"].(map[string]any)
+	if !ok {
+		t.Fatalf("result = %#v, want object", envelope["result"])
+	}
+	return result
+}

internal/bundle/digest.go

@@ -1,49 +1,19 @@
 package bundle
 
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-)
-
-var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
+import publicbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
 
 func ValidateDigest(value string) error {
-	if !digestPattern.MatchString(value) {
-		return fmt.Errorf("must be lowercase sha256:<64 hex>")
-	}
-	return nil
+	return publicbundle.ValidateDigest(value)
 }
 
 func FileDigest(data []byte) string {
-	sum := sha256.Sum256(data)
-	return "sha256:" + hex.EncodeToString(sum[:])
+	return publicbundle.FileDigest(data)
 }
 
 func BundleDigest(files []ManifestFile) string {
-	canonical := CanonicalFilePayload(files)
-	sum := sha256.Sum256([]byte(canonical))
-	return "sha256:" + hex.EncodeToString(sum[:])
+	return publicbundle.BundleDigest(files)
 }
 
 func CanonicalFilePayload(files []ManifestFile) string {
-	var builder strings.Builder
-	builder.WriteByte('[')
-	for index, file := range files {
-		if index > 0 {
-			builder.WriteByte(',')
-		}
-		builder.WriteString(`{"path":`)
-		builder.WriteString(strconv.Quote(file.Path))
-		builder.WriteString(`,"sha256":`)
-		builder.WriteString(strconv.Quote(file.SHA256))
-		builder.WriteString(`,"size":`)
-		builder.WriteString(strconv.FormatInt(file.Size, 10))
-		builder.WriteByte('}')
-	}
-	builder.WriteByte(']')
-	return builder.String()
+	return publicbundle.CanonicalFilePayload(files)
 }

internal/bundle/manifest.go

@@ -1,110 +1,32 @@
 package bundle
 
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"time"
-)
+import publicbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
 
-const ManifestName = "manifest.json"
+const ManifestName = publicbundle.ManifestName
 
-type Manifest struct {
-	SchemaVersion int            `json:"schema_version"`
-	ID            string         `json:"id"`
-	Digest        string         `json:"digest"`
-	Created       time.Time      `json:"created"`
-	Files         []ManifestFile `json:"files"`
-}
+const SchemaVersion = publicbundle.SchemaVersion
 
-type ManifestFile struct {
-	Path   string `json:"path"`
-	SHA256 string `json:"sha256"`
-	Size   int64  `json:"size"`
-}
+type Manifest = publicbundle.Manifest
+
+type ManifestFile = publicbundle.ManifestFile
 
 type Bundle struct {
 	RootRelativePath string
 	Manifest         Manifest
 }
 
-type rawManifest struct {
-	SchemaVersion *int              `json:"schema_version"`
-	ID            *string           `json:"id"`
-	Digest        *string           `json:"digest"`
-	Created       *string           `json:"created"`
-	Files         []rawManifestFile `json:"files"`
-}
-
-type rawManifestFile struct {
-	Path   *string `json:"path"`
-	SHA256 *string `json:"sha256"`
-	Size   *int64  `json:"size"`
-}
-
 func ParseManifest(data []byte) (Manifest, error) {
-	decoder := json.NewDecoder(bytes.NewReader(data))
-	var raw rawManifest
-	if err := decoder.Decode(&raw); err != nil {
-		return Manifest{}, fmt.Errorf("parse manifest: %w", err)
-	}
-	var extra any
-	if err := decoder.Decode(&extra); err != io.EOF {
-		return Manifest{}, fmt.Errorf("parse manifest: trailing data")
-	}
-
-	var manifest Manifest
-	if raw.SchemaVersion == nil {
-		return Manifest{}, fmt.Errorf("manifest schema_version is required")
-	}
-	manifest.SchemaVersion = *raw.SchemaVersion
-	if raw.ID == nil || *raw.ID == "" {
-		return Manifest{}, fmt.Errorf("manifest id is required")
-	}
-	manifest.ID = *raw.ID
-	if raw.Digest == nil || *raw.Digest == "" {
-		return Manifest{}, fmt.Errorf("manifest digest is required")
-	}
-	manifest.Digest = *raw.Digest
-	if raw.Created == nil || *raw.Created == "" {
-		return Manifest{}, fmt.Errorf("manifest created is required")
-	}
-	created, err := time.Parse(time.RFC3339, *raw.Created)
-	if err != nil {
-		return Manifest{}, fmt.Errorf("manifest created must be RFC3339: %w", err)
-	}
-	manifest.Created = created
-	if len(raw.Files) == 0 {
-		return Manifest{}, fmt.Errorf("manifest files is required")
-	}
-
-	for index, rawFile := range raw.Files {
-		file, err := parseManifestFile(index, rawFile)
-		if err != nil {
-			return Manifest{}, err
-		}
-		manifest.Files = append(manifest.Files, file)
-	}
-	if err := ValidateManifest(manifest); err != nil {
-		return Manifest{}, fmt.Errorf("manifest %w", err)
-	}
-	return manifest, nil
+	return publicbundle.ParseManifest(data)
 }
 
-func parseManifestFile(index int, raw rawManifestFile) (ManifestFile, error) {
-	if raw.Path == nil || *raw.Path == "" {
-		return ManifestFile{}, fmt.Errorf("manifest files[%d].path is required", index)
-	}
-	if raw.SHA256 == nil || *raw.SHA256 == "" {
-		return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256 is required", index)
-	}
-	if raw.Size == nil {
-		return ManifestFile{}, fmt.Errorf("manifest files[%d].size is required", index)
-	}
-	return ManifestFile{
-		Path:   *raw.Path,
-		SHA256: *raw.SHA256,
-		Size:   *raw.Size,
-	}, nil
+func MarshalManifest(manifest Manifest) ([]byte, error) {
+	return publicbundle.MarshalManifest(manifest)
+}
+
+func ValidateManifest(manifest Manifest) error {
+	return publicbundle.ValidateManifest(manifest)
+}
+
+func ValidateSourcePath(path string) error {
+	return publicbundle.ValidateSourcePath(path)
 }

internal/bundle/manifest_test.go

@@ -70,11 +70,19 @@ func TestParseManifestRejectsInvalidDigestFormat(t *testing.T) {
 
 func TestParseManifestRejectsUnsafeFilePaths(t *testing.T) {
 	tests := []string{
+		`"path": ""`,
+		`"path": "."`,
+		`"path": "./report.md"`,
 		`"path": "../report.md"`,
 		`"path": "/report.md"`,
 		`"path": "nested/../report.md"`,
+		`"path": "nested/./report.md"`,
+		`"path": "nested//report.md"`,
+		`"path": "nested\\report.md"`,
 		`"path": "manifest.json"`,
+		`"path": "nested/manifest.json"`,
 		`"path": "` + storage.StateFileName + `"`,
+		`"path": "nested/` + storage.StateFileName + `"`,
 	}
 	for _, replacement := range tests {
 		t.Run(replacement, func(t *testing.T) {
@@ -128,6 +136,16 @@ func TestValidateManifestRejectsInvalidManifest(t *testing.T) {
 			manifest.Digest = BundleDigest(manifest.Files)
 			return manifest
 		},
+		"nested manifest path": func(manifest Manifest) Manifest {
+			manifest.Files[0].Path = "nested/manifest.json"
+			manifest.Digest = BundleDigest(manifest.Files)
+			return manifest
+		},
+		"nested state path": func(manifest Manifest) Manifest {
+			manifest.Files[0].Path = "nested/" + storage.StateFileName
+			manifest.Digest = BundleDigest(manifest.Files)
+			return manifest
+		},
 		"duplicate path": func(manifest Manifest) Manifest {
 			manifest.Files[1].Path = manifest.Files[0].Path
 			manifest.Digest = BundleDigest(manifest.Files)

internal/bundle/validate.go

@@ -7,55 +7,6 @@ import (
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 )
 
-func ValidateSourcePath(path string) error {
-	if err := storage.ValidatePath(path); err != nil {
-		return err
-	}
-	switch path {
-	case ManifestName, storage.StateFileName:
-		return fmt.Errorf("%q is reserved", path)
-	}
-	return nil
-}
-
-func ValidateManifest(manifest Manifest) error {
-	if manifest.SchemaVersion != 1 {
-		return fmt.Errorf("schema_version must be 1")
-	}
-	if manifest.ID == "" {
-		return fmt.Errorf("id is required")
-	}
-	if err := ValidateDigest(manifest.Digest); err != nil {
-		return fmt.Errorf("digest: %w", err)
-	}
-	if manifest.Created.IsZero() {
-		return fmt.Errorf("created is required")
-	}
-	if len(manifest.Files) == 0 {
-		return fmt.Errorf("files is required")
-	}
-	seen := make(map[string]struct{}, len(manifest.Files))
-	for index, file := range manifest.Files {
-		if err := ValidateSourcePath(file.Path); err != nil {
-			return fmt.Errorf("files[%d].path: %w", index, err)
-		}
-		if err := ValidateDigest(file.SHA256); err != nil {
-			return fmt.Errorf("files[%d].sha256: %w", index, err)
-		}
-		if file.Size < 0 {
-			return fmt.Errorf("files[%d].size must be non-negative", index)
-		}
-		if _, exists := seen[file.Path]; exists {
-			return fmt.Errorf("files[%d].path duplicates %q", index, file.Path)
-		}
-		seen[file.Path] = struct{}{}
-	}
-	if actual := BundleDigest(manifest.Files); actual != manifest.Digest {
-		return fmt.Errorf("digest mismatch: got %s want %s", actual, manifest.Digest)
-	}
-	return nil
-}
-
 func Validate(ctx context.Context, backend storage.Backend, bundleRoot string) (Bundle, error) {
 	return validateAt(ctx, backend, bundleRoot, bundleRoot)
 }

internal/bundle/validate_test.go

@@ -65,6 +65,38 @@ func TestValidateRejectsSymlinkFile(t *testing.T) {
 	assertErrorContains(t, err, "regular file")
 }
 
+func TestValidateRejectsUnsafeManifestPaths(t *testing.T) {
+	tests := []string{
+		"",
+		".",
+		"./report.md",
+		"../report.md",
+		"/report.md",
+		"nested/../report.md",
+		"nested/./report.md",
+		"nested//report.md",
+		`nested\report.md`,
+		ManifestName,
+		storage.StateFileName,
+		"nested/" + ManifestName,
+		"nested/" + storage.StateFileName,
+	}
+	for _, path := range tests {
+		t.Run(path, func(t *testing.T) {
+			backend := validFakeBundle(t)
+			manifest := validFixtureManifest(t)
+			manifest.Files[0].Path = path
+			manifest.Digest = BundleDigest(manifest.Files)
+			writeManifest(t, backend, manifest)
+
+			_, err := Validate(context.Background(), backend, "")
+			if err == nil {
+				t.Fatal("Validate() error = nil, want unsafe path error")
+			}
+		})
+	}
+}
+
 func validFakeBundle(t *testing.T) *fake.Backend {
 	t.Helper()
 	backend := fake.New()

internal/cli/flags.go

@@ -0,0 +1,12 @@
+package cli
+
+import (
+	"flag"
+	"io"
+)
+
+func newFlagSet(name string, stderr io.Writer) *flag.FlagSet {
+	flags := flag.NewFlagSet(name, flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	return flags
+}

internal/cli/format.go

@@ -0,0 +1,22 @@
+package cli
+
+import (
+	"flag"
+	"fmt"
+	"io"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/app"
+)
+
+func addFormatFlag(flags *flag.FlagSet) *string {
+	return flags.String("format", string(app.OutputFormatText), "output format: text or json")
+}
+
+func parseOutputFormat(stderr io.Writer, command, raw string) (app.OutputFormat, bool) {
+	format := app.OutputFormat(raw)
+	if err := app.ValidateOutputFormat(format); err != nil {
+		fmt.Fprintf(stderr, "%s: %s --format: %s\n", app.Name, command, err)
+		return "", false
+	}
+	return app.NormalizeOutputFormat(format), true
+}

internal/cli/inspect.go

@@ -13,11 +13,18 @@ func inspectCommand(ctx context.Context, args []string, stdout, stderr io.Writer
 		printInspectHelp(stdout)
 		return exitOK
 	}
-	path, ok := parseOptionalPathArg(stderr, "inspect", args)
+	parsed, ok := parseSourceDiagnosticArgs(stderr, "inspect", args)
 	if !ok {
 		return exitUsage
 	}
-	if err := app.Inspect(ctx, app.InspectOptions{Path: path, Stdout: stdout}); err != nil {
+	if err := app.Inspect(ctx, app.InspectOptions{
+		Path:         parsed.Path,
+		ConfigPath:   parsed.ConfigPath,
+		PipelineID:   parsed.PipelineID,
+		BundlePath:   parsed.BundlePath,
+		Stdout:       stdout,
+		OutputFormat: parsed.OutputFormat,
+	}); err != nil {
 		return fail(stderr, err)
 	}
 	return exitOK
@@ -25,8 +32,16 @@ func inspectCommand(ctx context.Context, args []string, stdout, stderr io.Writer
 
 func printInspectHelp(w io.Writer) {
 	fmt.Fprint(w, `Usage:
-  distributor inspect <path>
+  distributor inspect [--format text|json] <path>
+  distributor inspect --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
 
-Print a normalized summary of local source bundles.
+Options:
+  --config <path>     Path to config file for configured source inspection
+  --pipeline <id>     Pipeline id to inspect in config mode
+  --bundle <path>     Source-root-relative bundle path to inspect
+  --format text|json  Output format
+
+Print a normalized summary of local source bundles or a configured pipeline
+source.
 `)
 }

internal/cli/manifest.go

@@ -0,0 +1,137 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/app"
+)
+
+func manifestCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 || args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
+		printManifestHelp(stdout)
+		return exitOK
+	}
+	switch args[0] {
+	case "create":
+		return manifestCreateCommand(ctx, args[1:], stdout, stderr)
+	default:
+		fmt.Fprintf(stderr, "%s: manifest unknown command %q\n\n", app.Name, args[0])
+		printManifestHelp(stderr)
+		return exitUsage
+	}
+}
+
+func manifestCreateCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+	if hasHelp(args) {
+		printManifestCreateHelp(stdout)
+		return exitOK
+	}
+	flags := newFlagSet("manifest create", stderr)
+	id := flags.String("id", "", "source bundle id")
+	created := flags.String("created", "", "source created timestamp")
+	overwrite := flags.Bool("overwrite", false, "replace an existing manifest.json")
+	formatFlag := addFormatFlag(flags)
+	var files repeatedFlag
+	flags.Var(&files, "file", "bundle-relative file to include")
+	flagArgs, positionalArgs, ok := splitManifestCreateArgs(stderr, args)
+	if !ok {
+		return exitUsage
+	}
+	if err := flags.Parse(flagArgs); err != nil {
+		return exitUsage
+	}
+	if len(positionalArgs) != 1 {
+		fmt.Fprintf(stderr, "%s: manifest create requires exactly one bundle path\n", app.Name)
+		return exitUsage
+	}
+	format, ok := parseOutputFormat(stderr, "manifest create", *formatFlag)
+	if !ok {
+		return exitUsage
+	}
+	err := app.ManifestCreate(ctx, app.ManifestCreateOptions{
+		Root:         positionalArgs[0],
+		ID:           *id,
+		Created:      *created,
+		Files:        []string(files),
+		Overwrite:    *overwrite,
+		Stdout:       stdout,
+		OutputFormat: format,
+	})
+	if err != nil {
+		return fail(stderr, err)
+	}
+	return exitOK
+}
+
+func splitManifestCreateArgs(stderr io.Writer, args []string) ([]string, []string, bool) {
+	var flagArgs []string
+	var positionalArgs []string
+	for index := 0; index < len(args); index++ {
+		arg := args[index]
+		switch arg {
+		case "--overwrite":
+			flagArgs = append(flagArgs, arg)
+		case "--id", "--created", "--file", "--format":
+			if index+1 >= len(args) {
+				fmt.Fprintf(stderr, "%s: manifest create %s requires a value\n", app.Name, arg)
+				return nil, nil, false
+			}
+			flagArgs = append(flagArgs, arg, args[index+1])
+			index++
+		default:
+			if strings.HasPrefix(arg, "--id=") ||
+				strings.HasPrefix(arg, "--created=") ||
+				strings.HasPrefix(arg, "--file=") ||
+				strings.HasPrefix(arg, "--format=") {
+				flagArgs = append(flagArgs, arg)
+				continue
+			}
+			if strings.HasPrefix(arg, "-") {
+				flagArgs = append(flagArgs, arg)
+				continue
+			}
+			positionalArgs = append(positionalArgs, arg)
+		}
+	}
+	return flagArgs, positionalArgs, true
+}
+
+type repeatedFlag []string
+
+func (f *repeatedFlag) String() string {
+	return fmt.Sprint([]string(*f))
+}
+
+func (f *repeatedFlag) Set(value string) error {
+	*f = append(*f, value)
+	return nil
+}
+
+func printManifestHelp(w io.Writer) {
+	fmt.Fprint(w, `Usage:
+  distributor manifest <command> [options]
+
+Commands:
+  create    Create a source bundle manifest
+
+Use "distributor manifest <command> --help" for command-specific help.
+`)
+}
+
+func printManifestCreateHelp(w io.Writer) {
+	fmt.Fprint(w, `Usage:
+  distributor manifest create <bundle-path> --id <bundle-id> [options]
+
+Options:
+  --id <bundle-id>     Source bundle id
+  --file <path>        Bundle-relative file to include; repeatable
+  --created <time>     RFC3339 source created timestamp
+  --overwrite          Replace an existing manifest.json
+  --format text|json   Output format
+
+Create manifest.json for a local source bundle directory.
+`)
+}

internal/cli/root.go

@@ -29,10 +29,14 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
 		return versionCommand(ctx, args[1:], stdout, stderr)
 	case "run":
 		return runCommand(ctx, args[1:], stdout, stderr)
+	case "serve":
+		return serveCommand(ctx, args[1:], stdout, stderr)
 	case "validate":
 		return validateCommand(ctx, args[1:], stdout, stderr)
 	case "inspect":
 		return inspectCommand(ctx, args[1:], stdout, stderr)
+	case "manifest":
+		return manifestCommand(ctx, args[1:], stdout, stderr)
 	default:
 		fmt.Fprintf(stderr, "%s: unknown command %q\n\n", app.Name, args[0])
 		printRootHelp(stderr)
@@ -49,8 +53,10 @@ Usage:
 Commands:
   version    Print version information
   run        Run configured distribution pipelines
+  serve      Run the HTTP upload server
   validate   Validate a source bundle or bundle tree
   inspect    Inspect bundles or distributor state
+  manifest   Create source bundle manifests
 
 Use "%s <command> --help" for command-specific help.
 `, app.Name, app.Name, app.Name)

internal/cli/root_test.go

@@ -3,15 +3,43 @@ package cli
 import (
 	"bytes"
 	"context"
+	"encoding/json"
+	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"gitea.maximumdirect.net/eric/distributor/internal/app"
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
+	producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
 )
 
+func decodeEnvelope(t *testing.T, stdout *bytes.Buffer) map[string]any {
+	t.Helper()
+	decoder := json.NewDecoder(strings.NewReader(stdout.String()))
+	var envelope map[string]any
+	if err := decoder.Decode(&envelope); err != nil {
+		t.Fatalf("decode JSON envelope: %v; stdout = %q", err, stdout.String())
+	}
+	var extra any
+	if err := decoder.Decode(&extra); err != io.EOF {
+		t.Fatalf("stdout contains more than one JSON document: %q", stdout.String())
+	}
+	return envelope
+}
+
+func envelopeResult(t *testing.T, envelope map[string]any) map[string]any {
+	t.Helper()
+	result, ok := envelope["result"].(map[string]any)
+	if !ok {
+		t.Fatalf("result = %#v, want object", envelope["result"])
+	}
+	return result
+}
+
 func TestExecuteRootHelp(t *testing.T) {
 	var stdout, stderr bytes.Buffer
 
@@ -44,6 +72,71 @@ func TestExecuteVersion(t *testing.T) {
 	}
 }
 
+func TestExecuteVersionJSON(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"version", "--format", "json"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	if envelope["command"] != "version" || envelope["ok"] != true {
+		t.Fatalf("envelope = %#v, want version ok", envelope)
+	}
+	result := envelopeResult(t, envelope)
+	if result["application"] != "distributor" || result["version"] != "dev" {
+		t.Fatalf("result = %#v, want application/version", result)
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestExecuteServeParsesConfig(t *testing.T) {
+	originalServeApp := serveApp
+	defer func() {
+		serveApp = originalServeApp
+	}()
+	var gotOptions app.ServeOptions
+	serveApp = func(_ context.Context, options app.ServeOptions) error {
+		gotOptions = options
+		return nil
+	}
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"serve", "--config", "config.yml"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	if gotOptions.ConfigPath != "config.yml" {
+		t.Fatalf("ConfigPath = %q, want config.yml", gotOptions.ConfigPath)
+	}
+	if stdout.Len() != 0 {
+		t.Fatalf("stdout = %q, want empty", stdout.String())
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestExecuteRejectsInvalidFormat(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"version", "--format", "xml"}, &stdout, &stderr)
+
+	if code != exitUsage {
+		t.Fatalf("exit code = %d, want %d", code, exitUsage)
+	}
+	if stdout.Len() != 0 {
+		t.Fatalf("stdout = %q, want empty", stdout.String())
+	}
+	if !strings.Contains(stderr.String(), "format must be text or json") {
+		t.Fatalf("stderr = %q, want invalid format error", stderr.String())
+	}
+}
+
 func TestExecuteValidate(t *testing.T) {
 	var stdout, stderr bytes.Buffer
 
@@ -57,6 +150,63 @@ func TestExecuteValidate(t *testing.T) {
 	}
 }
 
+func TestExecuteValidateJSON(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"validate", "--format", "json", filepath.Join("..", "bundle", "testdata", "valid_bundle")}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	if envelope["command"] != "validate" || envelope["ok"] != true {
+		t.Fatalf("envelope = %#v, want validate ok", envelope)
+	}
+	result := envelopeResult(t, envelope)
+	if result["bundle_count"] != float64(1) {
+		t.Fatalf("result = %#v, want one bundle", result)
+	}
+}
+
+func TestExecuteValidateConfiguredSource(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"validate", "--config", configPath, "--pipeline", "reports"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	if got, want := stdout.String(), "Validated 1 bundle(s) for pipeline reports source local\n"; got != want {
+		t.Fatalf("stdout = %q, want %q", got, want)
+	}
+}
+
+func TestExecuteValidateConfiguredSourceJSON(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "daily", testutil.BundleOptions{ID: "reports.daily"})
+	configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"validate", "--config", configPath, "--pipeline", "reports", "--bundle", "daily", "--format", "json"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	if envelope["command"] != "validate" || envelope["ok"] != true {
+		t.Fatalf("envelope = %#v, want validate ok", envelope)
+	}
+	result := envelopeResult(t, envelope)
+	if result["pipeline_id"] != "reports" || result["source_backend"] != "local" || result["bundle_count"] != float64(1) {
+		t.Fatalf("result = %#v, want configured source metadata", result)
+	}
+}
+
 func TestExecuteValidateArgs(t *testing.T) {
 	validPath := filepath.Join("..", "bundle", "testdata", "valid_bundle")
 	tests := []struct {
@@ -84,6 +234,36 @@ func TestExecuteValidateArgs(t *testing.T) {
 			wantCode:   exitUsage,
 			wantStderr: "accepts at most one path",
 		},
+		{
+			name:       "path plus config",
+			args:       []string{"validate", "--config", "config.yml", "--pipeline", "reports", validPath},
+			wantCode:   exitUsage,
+			wantStderr: "does not accept a local path",
+		},
+		{
+			name:       "pipeline without config",
+			args:       []string{"validate", "--pipeline", "reports"},
+			wantCode:   exitUsage,
+			wantStderr: "requires --config",
+		},
+		{
+			name:       "bundle without config",
+			args:       []string{"validate", "--bundle", "daily"},
+			wantCode:   exitUsage,
+			wantStderr: "requires --config",
+		},
+		{
+			name:       "config without pipeline",
+			args:       []string{"validate", "--config", "config.yml"},
+			wantCode:   exitUsage,
+			wantStderr: "requires --pipeline",
+		},
+		{
+			name:       "invalid format",
+			args:       []string{"validate", "--format", "xml", validPath},
+			wantCode:   exitUsage,
+			wantStderr: "format must be text or json",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -115,6 +295,56 @@ func TestExecuteInspect(t *testing.T) {
 	}
 }
 
+func TestExecuteInspectJSON(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"inspect", "--format", "json", filepath.Join("..", "bundle", "testdata", "valid_bundle")}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	if envelope["command"] != "inspect" || envelope["ok"] != true {
+		t.Fatalf("envelope = %#v, want inspect ok", envelope)
+	}
+	result := envelopeResult(t, envelope)
+	bundles, ok := result["bundles"].([]any)
+	if !ok || len(bundles) != 1 {
+		t.Fatalf("bundles = %#v, want one bundle", result["bundles"])
+	}
+	bundle, ok := bundles[0].(map[string]any)
+	if !ok {
+		t.Fatalf("bundle = %#v, want object", bundles[0])
+	}
+	if bundle["id"] != "weather.daily.brentwood.2026-05-30" || bundle["file_count"] != float64(2) || bundle["total_size"] != float64(24) {
+		t.Fatalf("bundle = %#v, want normalized metadata", bundle)
+	}
+}
+
+func TestExecuteInspectConfiguredSource(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "daily", testutil.BundleOptions{ID: "reports.daily"})
+	configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"inspect", "--config", configPath, "--pipeline", "reports"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	for _, want := range []string{
+		"Pipeline: reports",
+		"Source: local",
+		"path=daily",
+		"id=reports.daily",
+	} {
+		if !strings.Contains(stdout.String(), want) {
+			t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
+		}
+	}
+}
+
 func TestExecuteInspectArgs(t *testing.T) {
 	validPath := filepath.Join("..", "bundle", "testdata", "valid_bundle")
 	tests := []struct {
@@ -142,6 +372,36 @@ func TestExecuteInspectArgs(t *testing.T) {
 			wantCode:   exitUsage,
 			wantStderr: "accepts at most one path",
 		},
+		{
+			name:       "path plus config",
+			args:       []string{"inspect", "--config", "config.yml", "--pipeline", "reports", validPath},
+			wantCode:   exitUsage,
+			wantStderr: "does not accept a local path",
+		},
+		{
+			name:       "pipeline without config",
+			args:       []string{"inspect", "--pipeline", "reports"},
+			wantCode:   exitUsage,
+			wantStderr: "requires --config",
+		},
+		{
+			name:       "bundle without config",
+			args:       []string{"inspect", "--bundle", "daily"},
+			wantCode:   exitUsage,
+			wantStderr: "requires --config",
+		},
+		{
+			name:       "config without pipeline",
+			args:       []string{"inspect", "--config", "config.yml"},
+			wantCode:   exitUsage,
+			wantStderr: "requires --pipeline",
+		},
+		{
+			name:       "invalid format",
+			args:       []string{"inspect", "--format", "xml", validPath},
+			wantCode:   exitUsage,
+			wantStderr: "format must be text or json",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -160,6 +420,206 @@ func TestExecuteInspectArgs(t *testing.T) {
 	}
 }
 
+func TestExecuteManifestCreateExplicitFiles(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "b.txt", "bravo")
+	writeCLIFile(t, root, "nested/a.txt", "alpha")
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{
+		"manifest", "create", root,
+		"--id", "reports.explicit",
+		"--created", "2026-06-01T11:00:00Z",
+		"--file", "b.txt",
+		"--file", "nested/a.txt",
+	}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	for _, want := range []string{
+		"created manifest.json",
+		"bundle: reports.explicit",
+		"files: 2",
+		"digest: sha256:",
+	} {
+		if !strings.Contains(stdout.String(), want) {
+			t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
+		}
+	}
+	manifest, err := producerbundle.LoadManifest(root)
+	if err != nil {
+		t.Fatalf("LoadManifest() error = %v", err)
+	}
+	if got, want := manifestPaths(manifest), []string{"b.txt", "nested/a.txt"}; !equalStrings(got, want) {
+		t.Fatalf("manifest paths = %v, want %v", got, want)
+	}
+	if err := producerbundle.ValidateBundle(root, manifest); err != nil {
+		t.Fatalf("ValidateBundle() error = %v", err)
+	}
+	var validateStdout, validateStderr bytes.Buffer
+	validateCode := Execute(context.Background(), []string{"validate", root}, &validateStdout, &validateStderr)
+	if validateCode != exitOK {
+		t.Fatalf("validate exit code = %d, want %d; stderr = %q", validateCode, exitOK, validateStderr.String())
+	}
+}
+
+func TestExecuteManifestCreateScansBundle(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "z.txt", "zulu")
+	writeCLIFile(t, root, ".env", "dotfile")
+	writeCLIFile(t, root, "nested/report.md", "# Report\n")
+	writeCLIFile(t, root, storage.StateFileName, "destination state")
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.scan"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	manifest, err := producerbundle.LoadManifest(root)
+	if err != nil {
+		t.Fatalf("LoadManifest() error = %v", err)
+	}
+	if got, want := manifestPaths(manifest), []string{".env", "nested/report.md", "z.txt"}; !equalStrings(got, want) {
+		t.Fatalf("manifest paths = %v, want %v", got, want)
+	}
+}
+
+func TestExecuteManifestCreateJSON(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "report.md", "# Report\n")
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.json", "--file", "report.md", "--format", "json"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	if envelope["command"] != "manifest create" || envelope["ok"] != true {
+		t.Fatalf("envelope = %#v, want manifest create ok", envelope)
+	}
+	result := envelopeResult(t, envelope)
+	if result["id"] != "reports.json" || result["file_count"] != float64(1) {
+		t.Fatalf("result = %#v, want manifest summary", result)
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestExecuteManifestCreateOverwrite(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "report.md", "old\n")
+	var stdout, stderr bytes.Buffer
+	code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.old", "--file", "report.md"}, &stdout, &stderr)
+	if code != exitOK {
+		t.Fatalf("initial exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	writeCLIFile(t, root, "report.md", "new\n")
+
+	stdout.Reset()
+	stderr.Reset()
+	code = Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.new", "--file", "report.md"}, &stdout, &stderr)
+	if code != exitError {
+		t.Fatalf("overwrite exit code = %d, want %d", code, exitError)
+	}
+	if !strings.Contains(stderr.String(), "write manifest") {
+		t.Fatalf("stderr = %q, want write manifest error", stderr.String())
+	}
+	manifest, err := producerbundle.LoadManifest(root)
+	if err != nil {
+		t.Fatalf("LoadManifest() error = %v", err)
+	}
+	if manifest.ID != "reports.old" {
+		t.Fatalf("manifest id = %q, want reports.old", manifest.ID)
+	}
+
+	stdout.Reset()
+	stderr.Reset()
+	code = Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.new", "--file", "report.md", "--overwrite"}, &stdout, &stderr)
+	if code != exitOK {
+		t.Fatalf("overwrite exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	manifest, err = producerbundle.LoadManifest(root)
+	if err != nil {
+		t.Fatalf("LoadManifest() error = %v", err)
+	}
+	if manifest.ID != "reports.new" {
+		t.Fatalf("manifest id = %q, want reports.new", manifest.ID)
+	}
+}
+
+func TestExecuteManifestCreateRejectsSymlink(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "target.md", "# Report\n")
+	if err := os.Symlink("target.md", filepath.Join(root, "link.md")); err != nil {
+		t.Skipf("symlink unavailable: %v", err)
+	}
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.link", "--file", "link.md"}, &stdout, &stderr)
+
+	if code != exitError {
+		t.Fatalf("exit code = %d, want %d", code, exitError)
+	}
+	if !strings.Contains(stderr.String(), "regular file") {
+		t.Fatalf("stderr = %q, want regular file error", stderr.String())
+	}
+}
+
+func TestExecuteManifestCreateArgs(t *testing.T) {
+	root := t.TempDir()
+	writeCLIFile(t, root, "report.md", "# Report\n")
+	tests := []struct {
+		name       string
+		args       []string
+		wantCode   int
+		wantStderr string
+	}{
+		{
+			name:       "missing path",
+			args:       []string{"manifest", "create", "--id", "reports.missing"},
+			wantCode:   exitUsage,
+			wantStderr: "requires exactly one bundle path",
+		},
+		{
+			name:       "missing id",
+			args:       []string{"manifest", "create", root},
+			wantCode:   exitError,
+			wantStderr: "requires --id",
+		},
+		{
+			name:       "bad created",
+			args:       []string{"manifest", "create", root, "--id", "reports.bad", "--created", "June 1"},
+			wantCode:   exitError,
+			wantStderr: "created must be RFC3339",
+		},
+		{
+			name:       "bad format",
+			args:       []string{"manifest", "create", root, "--id", "reports.bad", "--format", "xml"},
+			wantCode:   exitUsage,
+			wantStderr: "format must be text or json",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var stdout, stderr bytes.Buffer
+			code := Execute(context.Background(), tt.args, &stdout, &stderr)
+			if code != tt.wantCode {
+				t.Fatalf("exit code = %d, want %d; stderr = %q", code, tt.wantCode, stderr.String())
+			}
+			if stdout.Len() != 0 {
+				t.Fatalf("stdout = %q, want empty", stdout.String())
+			}
+			if !strings.Contains(stderr.String(), tt.wantStderr) {
+				t.Fatalf("stderr = %q, want substring %q", stderr.String(), tt.wantStderr)
+			}
+		})
+	}
+}
+
 func TestExecuteRunDryRun(t *testing.T) {
 	sourceRoot := t.TempDir()
 	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
@@ -172,14 +632,323 @@ func TestExecuteRunDryRun(t *testing.T) {
 	if code != exitOK {
 		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
 	}
-	if !strings.Contains(stdout.String(), "action=publish_new") {
-		t.Fatalf("stdout = %q, want config summary", stdout.String())
+	wantStdout := "Configured pipelines: 1\n" +
+		"- pipeline=reports source=local bundles=1 destinations=archive\n" +
+		"  - bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt reason=\"destination state is absent\"\n" +
+		"Final status: ok planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true fixed_path=0\n"
+	if got := stdout.String(); got != wantStdout {
+		t.Fatalf("stdout = %q, want %q", got, wantStdout)
 	}
 	if stderr.Len() != 0 {
 		t.Fatalf("stderr = %q, want empty", stderr.String())
 	}
 }
 
+func TestExecuteRunJSONDryRun(t *testing.T) {
+	sourceRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, t.TempDir())
+
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run", "--format", "json"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	if envelope["command"] != "run" || envelope["ok"] != true {
+		t.Fatalf("envelope = %#v, want run ok", envelope)
+	}
+	result := envelopeResult(t, envelope)
+	if result["dry_run"] != true {
+		t.Fatalf("result = %#v, want dry_run true", result)
+	}
+	pipelines, ok := result["pipelines"].([]any)
+	if !ok || len(pipelines) != 1 {
+		t.Fatalf("pipelines = %#v, want one pipeline", result["pipelines"])
+	}
+	pipeline, ok := pipelines[0].(map[string]any)
+	if !ok || pipeline["id"] != "reports" || pipeline["source_backend"] != "local" || pipeline["bundle_count"] != float64(1) {
+		t.Fatalf("pipeline = %#v, want reports/local summary", pipelines[0])
+	}
+	actions, ok := result["actions"].([]any)
+	if !ok || len(actions) != 1 {
+		t.Fatalf("actions = %#v, want one action", result["actions"])
+	}
+	action, ok := actions[0].(map[string]any)
+	if !ok || action["action"] != "publish_new" {
+		t.Fatalf("action = %#v, want publish_new", actions[0])
+	}
+	outputs, ok := action["outputs"].([]any)
+	if !ok || len(outputs) != 2 {
+		t.Fatalf("outputs = %#v, want source outputs", action["outputs"])
+	}
+	summary, ok := result["summary"].(map[string]any)
+	if !ok || summary["status"] != "ok" || summary["planned"] != float64(1) || summary["publish_new"] != float64(1) || summary["dry_run"] != true {
+		t.Fatalf("summary = %#v, want ok dry-run publish counters", result["summary"])
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestExecuteRunJSONDryRunReportsFixedPathMapping(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	configPath := filepath.Join(t.TempDir(), "config.yml")
+	if err := os.WriteFile(configPath, []byte(`
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: latest
+        backend: local
+        path: `+destinationRoot+`
+        path_mapping:
+          mode: fixed
+`), 0o600); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run", "--format", "json"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	warnings, ok := envelope["warnings"].([]any)
+	if !ok || len(warnings) != 1 {
+		t.Fatalf("warnings = %#v, want one fixed-path warning", envelope["warnings"])
+	}
+	warning, ok := warnings[0].(map[string]any)
+	if !ok || !strings.Contains(fmt.Sprint(warning["message"]), "path_mapping=fixed candidates=1 selected_bundle=.") {
+		t.Fatalf("warning = %#v, want fixed-path selection warning", warnings[0])
+	}
+	result := envelopeResult(t, envelope)
+	actions, ok := result["actions"].([]any)
+	if !ok || len(actions) != 1 {
+		t.Fatalf("actions = %#v, want one action", result["actions"])
+	}
+	action, ok := actions[0].(map[string]any)
+	if !ok || action["path_mapping"] != "fixed" || action["destination_path"] != "." || action["action"] != "publish_new" {
+		t.Fatalf("action = %#v, want fixed publish_new at root", actions[0])
+	}
+	summary, ok := result["summary"].(map[string]any)
+	if !ok || summary["fixed_path"] != float64(1) {
+		t.Fatalf("summary = %#v, want fixed_path 1", result["summary"])
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestExecuteRunJSONDryRunReportsLinks(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	configPath := filepath.Join(t.TempDir(), "config.yml")
+	if err := os.WriteFile(configPath, []byte(`
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: web
+        backend: local
+        path: `+destinationRoot+`
+        links:
+          base_url: https://reports.example.com/archive
+          primary: source
+`), 0o600); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run", "--format", "json"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	result := envelopeResult(t, envelope)
+	actions, ok := result["actions"].([]any)
+	if !ok || len(actions) != 1 {
+		t.Fatalf("actions = %#v, want one action", result["actions"])
+	}
+	action, ok := actions[0].(map[string]any)
+	if !ok || action["primary_url"] != "https://reports.example.com/archive/report.md" {
+		t.Fatalf("action = %#v, want primary URL", actions[0])
+	}
+	outputs, ok := action["outputs"].([]any)
+	if !ok || len(outputs) != 2 {
+		t.Fatalf("outputs = %#v, want two outputs", action["outputs"])
+	}
+	output, ok := outputs[0].(map[string]any)
+	if !ok || output["url"] != "https://reports.example.com/archive/report.md" {
+		t.Fatalf("output = %#v, want output URL", outputs[0])
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestExecuteRunJSONWarningsAreStructured(t *testing.T) {
+	name := "DISTRIBUTOR_TEST_CLI_JSON_SECRET"
+	t.Setenv(name, "process-value")
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	secretsRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	if err := os.WriteFile(filepath.Join(secretsRoot, name), []byte("secret-value\n"), 0o600); err != nil {
+		t.Fatalf("write secret: %v", err)
+	}
+	configPath := filepath.Join(t.TempDir(), "config.yml")
+	if err := os.WriteFile(configPath, []byte(`
+secrets:
+  directory: `+secretsRoot+`
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: archive
+        backend: local
+        path: `+destinationRoot+`
+`), 0o600); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+
+	var stdout, stderr bytes.Buffer
+	code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run", "--format", "json"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	warnings, ok := envelope["warnings"].([]any)
+	if !ok || len(warnings) != 1 {
+		t.Fatalf("warnings = %#v, want one warning", envelope["warnings"])
+	}
+	warning, ok := warnings[0].(map[string]any)
+	if !ok || !strings.Contains(fmt.Sprint(warning["message"]), name) {
+		t.Fatalf("warning = %#v, want secret name", warnings[0])
+	}
+	if strings.Contains(stdout.String(), "Warning:") || strings.Contains(stdout.String(), "process-value") || strings.Contains(stdout.String(), "secret-value") {
+		t.Fatalf("stdout exposed text warning or secret values: %q", stdout.String())
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestExecuteRunJSONFatalSetupErrorWritesNoJSON(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"run", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--format", "json"}, &stdout, &stderr)
+
+	if code != exitError {
+		t.Fatalf("exit code = %d, want %d", code, exitError)
+	}
+	if stdout.Len() != 0 {
+		t.Fatalf("stdout = %q, want empty", stdout.String())
+	}
+	if !strings.Contains(stderr.String(), "no such file or directory") {
+		t.Fatalf("stderr = %q, want setup error", stderr.String())
+	}
+}
+
+func TestExecuteRunJSONPartialFailure(t *testing.T) {
+	sourceRoot := t.TempDir()
+	firstDestination := t.TempDir()
+	secondDestination := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
+		t.Fatalf("write unmanaged file: %v", err)
+	}
+	configPath := filepath.Join(t.TempDir(), "config.yml")
+	if err := os.WriteFile(configPath, []byte(`
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: `+sourceRoot+`
+    destinations:
+      - id: archive-one
+        backend: local
+        path: `+firstDestination+`
+      - id: archive-two
+        backend: local
+        path: `+secondDestination+`
+`), 0o600); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+
+	var stdout, stderr bytes.Buffer
+	code := Execute(context.Background(), []string{"run", "--config", configPath, "--format", "json"}, &stdout, &stderr)
+
+	if code != exitError {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitError, stderr.String())
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty for partial JSON result", stderr.String())
+	}
+	envelope := decodeEnvelope(t, &stdout)
+	if envelope["command"] != "run" || envelope["ok"] != false {
+		t.Fatalf("envelope = %#v, want failed run envelope", envelope)
+	}
+	errors, ok := envelope["errors"].([]any)
+	if !ok || len(errors) != 1 {
+		t.Fatalf("errors = %#v, want one error", envelope["errors"])
+	}
+	result := envelopeResult(t, envelope)
+	summary, ok := result["summary"].(map[string]any)
+	if !ok || summary["status"] != "failed" || summary["failed"] != float64(1) {
+		t.Fatalf("summary = %#v, want failed summary", result["summary"])
+	}
+	actions, ok := result["actions"].([]any)
+	if !ok || len(actions) != 2 {
+		t.Fatalf("actions = %#v, want two actions", result["actions"])
+	}
+	if _, err := os.Stat(filepath.Join(secondDestination, storage.StateFileName)); err != nil {
+		t.Fatalf("second destination state stat error = %v", err)
+	}
+}
+
+func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
+	if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil {
+		t.Fatalf("write unmanaged file: %v", err)
+	}
+	configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
+
+	var stdout, stderr bytes.Buffer
+	code := Execute(context.Background(), []string{"run", "--config", configPath, "--force", "--dry-run"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	if !strings.Contains(stdout.String(), "action=force_replace") {
+		t.Fatalf("stdout = %q, want force_replace", stdout.String())
+	}
+	if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); err != nil {
+		t.Fatalf("unmanaged file stat error = %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(err) {
+		t.Fatalf("state stat error = %v, want not exist", err)
+	}
+}
+
 func TestExecuteRunRejectsExtraPositionalArgs(t *testing.T) {
 	var stdout, stderr bytes.Buffer
 
@@ -223,3 +992,34 @@ func TestUnknownCommandIsUsageError(t *testing.T) {
 		t.Fatalf("stderr = %q, want unknown command error", stderr.String())
 	}
 }
+
+func manifestPaths(manifest producerbundle.Manifest) []string {
+	paths := make([]string, 0, len(manifest.Files))
+	for _, file := range manifest.Files {
+		paths = append(paths, file.Path)
+	}
+	return paths
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for index := range a {
+		if a[index] != b[index] {
+			return false
+		}
+	}
+	return true
+}
+
+func writeCLIFile(t *testing.T, root, relative, body string) {
+	t.Helper()
+	path := filepath.Join(root, filepath.FromSlash(relative))
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
+	}
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}

internal/cli/run.go

@@ -2,7 +2,6 @@ package cli
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"io"
 
@@ -15,22 +14,32 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
 		return exitOK
 	}
 
-	flags := flag.NewFlagSet("run", flag.ContinueOnError)
-	flags.SetOutput(stderr)
+	flags := newFlagSet("run", stderr)
 	configPath := flags.String("config", "", "path to config file")
 	dryRun := flags.Bool("dry-run", false, "load and validate config without publishing")
+	force := flags.Bool("force", false, "allow explicit destructive replacement for supported conflicts")
+	formatFlag := addFormatFlag(flags)
 	if err := flags.Parse(args); err != nil {
 		return exitUsage
 	}
 	if rejectPositionalArgs(stderr, "run", flags.Args()) {
 		return exitUsage
 	}
+	format, ok := parseOutputFormat(stderr, "run", *formatFlag)
+	if !ok {
+		return exitUsage
+	}
 
 	if err := app.Run(ctx, app.RunOptions{
-		ConfigPath: *configPath,
-		DryRun:     *dryRun,
-		Stdout:     stdout,
+		ConfigPath:   *configPath,
+		DryRun:       *dryRun,
+		Force:        *force,
+		Stdout:       stdout,
+		OutputFormat: format,
 	}); err != nil {
+		if app.IsJSONOutput(format) && app.IsPartialResultError(err) {
+			return exitError
+		}
 		return fail(stderr, err)
 	}
 	return exitOK
@@ -38,13 +47,16 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
 
 func printRunHelp(w io.Writer) {
 	fmt.Fprint(w, `Usage:
-  distributor run --config <path> --dry-run
+  distributor run --config <path> [--dry-run] [--force] [--format text|json]
 
 Options:
   --config <path>  Path to config file
   --dry-run        Load and validate config without publishing
+  --force          Allow explicit destructive replacement for supported conflicts
+  --format text|json
+                   Output format
 
-Run discovers local source bundles, plans each configured destination, publishes
+Run discovers configured source bundles, plans each destination, publishes
 selected outputs unless --dry-run is set, and prints a final status summary.
 `)
 }

internal/cli/serve.go

@@ -0,0 +1,44 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/app"
+)
+
+var serveApp = app.Serve
+
+func serveCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+	if hasHelp(args) {
+		printServeHelp(stdout)
+		return exitOK
+	}
+
+	flags := newFlagSet("serve", stderr)
+	configPath := flags.String("config", "", "path to config file")
+	if err := flags.Parse(args); err != nil {
+		return exitUsage
+	}
+	if rejectPositionalArgs(stderr, "serve", flags.Args()) {
+		return exitUsage
+	}
+
+	if err := serveApp(ctx, app.ServeOptions{ConfigPath: *configPath}); err != nil {
+		return fail(stderr, err)
+	}
+	return exitOK
+}
+
+func printServeHelp(w io.Writer) {
+	fmt.Fprint(w, `Usage:
+  distributor serve --config <path>
+
+Options:
+  --config <path>  Path to config file
+
+Serve loads configured HTTP upload sources, resolves upload tokens through the
+configured secret environment, and starts the HTTP upload API.
+`)
+}

internal/cli/source_mode.go

@@ -0,0 +1,65 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/app"
+)
+
+type sourceDiagnosticArgs struct {
+	Path         string
+	ConfigPath   string
+	PipelineID   string
+	BundlePath   string
+	OutputFormat app.OutputFormat
+}
+
+func parseSourceDiagnosticArgs(stderr io.Writer, command string, args []string) (sourceDiagnosticArgs, bool) {
+	flags := newFlagSet(command, stderr)
+	configPath := flags.String("config", "", "path to config file")
+	pipelineID := flags.String("pipeline", "", "pipeline id")
+	bundlePath := flags.String("bundle", "", "source-root-relative bundle path")
+	formatFlag := addFormatFlag(flags)
+	if err := flags.Parse(args); err != nil {
+		return sourceDiagnosticArgs{}, false
+	}
+	format, ok := parseOutputFormat(stderr, command, *formatFlag)
+	if !ok {
+		return sourceDiagnosticArgs{}, false
+	}
+	path, ok := parseOptionalPathArg(stderr, command, flags.Args())
+	if !ok {
+		return sourceDiagnosticArgs{}, false
+	}
+	if !validateInspectModeOK(stderr, command, path, *configPath, *pipelineID, *bundlePath) {
+		return sourceDiagnosticArgs{}, false
+	}
+	return sourceDiagnosticArgs{
+		Path:         path,
+		ConfigPath:   *configPath,
+		PipelineID:   *pipelineID,
+		BundlePath:   *bundlePath,
+		OutputFormat: format,
+	}, true
+}
+
+func validateInspectModeOK(stderr io.Writer, command, path, configPath, pipelineID, bundlePath string) bool {
+	configMode := configPath != "" || pipelineID != "" || bundlePath != ""
+	if !configMode {
+		return true
+	}
+	if path != "" {
+		fmt.Fprintf(stderr, "distributor: %s does not accept a local path with --config, --pipeline, or --bundle\n", command)
+		return false
+	}
+	if configPath == "" {
+		fmt.Fprintf(stderr, "distributor: %s requires --config when --pipeline or --bundle is set\n", command)
+		return false
+	}
+	if pipelineID == "" {
+		fmt.Fprintf(stderr, "distributor: %s requires --pipeline in config mode\n", command)
+		return false
+	}
+	return true
+}

internal/cli/validate.go

@@ -13,11 +13,18 @@ func validateCommand(ctx context.Context, args []string, stdout, stderr io.Write
 		printValidateHelp(stdout)
 		return exitOK
 	}
-	path, ok := parseOptionalPathArg(stderr, "validate", args)
+	parsed, ok := parseSourceDiagnosticArgs(stderr, "validate", args)
 	if !ok {
 		return exitUsage
 	}
-	if err := app.Validate(ctx, app.ValidateOptions{Path: path, Stdout: stdout}); err != nil {
+	if err := app.Validate(ctx, app.ValidateOptions{
+		Path:         parsed.Path,
+		ConfigPath:   parsed.ConfigPath,
+		PipelineID:   parsed.PipelineID,
+		BundlePath:   parsed.BundlePath,
+		Stdout:       stdout,
+		OutputFormat: parsed.OutputFormat,
+	}); err != nil {
 		return fail(stderr, err)
 	}
 	return exitOK
@@ -25,8 +32,16 @@ func validateCommand(ctx context.Context, args []string, stdout, stderr io.Write
 
 func printValidateHelp(w io.Writer) {
 	fmt.Fprint(w, `Usage:
-  distributor validate <path>
+  distributor validate [--format text|json] <path>
+  distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
 
-Validate a local source bundle directory or a tree containing source bundles.
+Options:
+  --config <path>     Path to config file for configured source validation
+  --pipeline <id>     Pipeline id to validate in config mode
+  --bundle <path>     Source-root-relative bundle path to validate
+  --format text|json  Output format
+
+Validate a local source bundle directory, a local source bundle tree, or a
+configured pipeline source.
 `)
 }

internal/cli/version.go

@@ -13,16 +13,43 @@ func versionCommand(_ context.Context, args []string, stdout, stderr io.Writer)
 		printVersionHelp(stdout)
 		return exitOK
 	}
-	if rejectExtraArgs(stderr, "version", args) {
+	flags := newFlagSet("version", stderr)
+	formatFlag := addFormatFlag(flags)
+	if err := flags.Parse(args); err != nil {
 		return exitUsage
 	}
+	if rejectExtraArgs(stderr, "version", flags.Args()) {
+		return exitUsage
+	}
+	format, ok := parseOutputFormat(stderr, "version", *formatFlag)
+	if !ok {
+		return exitUsage
+	}
+	if app.IsJSONOutput(format) {
+		err := app.WriteJSONEnvelope(stdout, "version", true, nil, versionResult{
+			Application: app.Name,
+			Version:     app.Version,
+		}, nil)
+		if err != nil {
+			return fail(stderr, err)
+		}
+		return exitOK
+	}
 	fmt.Fprintln(stdout, app.VersionString())
 	return exitOK
 }
 
+type versionResult struct {
+	Application string `json:"application"`
+	Version     string `json:"version"`
+}
+
 func printVersionHelp(w io.Writer) {
 	fmt.Fprint(w, `Usage:
-  distributor version
+  distributor version [--format text|json]
+
+Options:
+  --format text|json  Output format
 
 Print version information.
 `)

internal/config/backend_view.go

@@ -0,0 +1,50 @@
+package config
+
+type backendView struct {
+	Backend   string
+	Host      string
+	User      string
+	Port      int
+	Path      string
+	Endpoint  string
+	Bucket    string
+	Prefix    string
+	Region    string
+	ForcePath *bool
+	Creds     Credentials
+	SSH       SSH
+}
+
+func backendViewFromSource(source Backend) backendView {
+	return backendView{
+		Backend:   source.Backend,
+		Host:      source.Host,
+		User:      source.User,
+		Port:      source.Port,
+		Path:      source.Path,
+		Endpoint:  source.Endpoint,
+		Bucket:    source.Bucket,
+		Prefix:    source.Prefix,
+		Region:    source.Region,
+		ForcePath: source.ForcePath,
+		Creds:     source.Creds,
+		SSH:       source.SSH,
+	}
+}
+
+func backendViewFromDestination(destination Destination) backendView {
+	return backendView{
+		Backend:   destination.Backend,
+		Host:      destination.Host,
+		User:      destination.User,
+		Port:      destination.Port,
+		Path:      destination.Path,
+		Endpoint:  destination.Endpoint,
+		Bucket:    destination.Bucket,
+		Prefix:    destination.Prefix,
+		Region:    destination.Region,
+		ForcePath: destination.ForcePath,
+		Creds:     destination.Creds,
+		SSH:       destination.SSH,
+	}
+}