Create a cleanup roadmap to address the items identified in the audit

README.md

@@ -2,7 +2,10 @@
 
 `distributor` validates manifested report bundles and publishes selected source or generated artifacts to configured destinations.
 
-It is a local-first CLI with SSH/SFTP and S3-compatible storage support: source bundles can be read from local or remote storage, destinations can be local directories or remote paths, and Markdown files can be rendered to HTML sidecars or `index.html`.
+It is a local-first CLI with SSH/SFTP, S3-compatible storage, and HTTP upload
+support: source bundles can be read from local or remote storage, pushed to the
+upload API, published to local directories or remote paths, and rendered from
+Markdown to HTML sidecars or `index.html`.
 
 Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build, write, parse, and validate complete local source bundles with the same manifest contract used by `distributor`.
 

docs/cli.md

@@ -14,6 +14,7 @@ This discovers the example source bundle and publishes source files to `workspac
 distributor [--help]
 distributor version [--format text|json]
 distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
+distributor serve [--config <path>]
 distributor validate [--format text|json] <path>
 distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
 distributor inspect [--format text|json] <path>
@@ -23,11 +24,12 @@ distributor manifest create <bundle-path> --id <bundle-id> [options]
 
 - `version`: prints the application name and version. Development builds print `distributor dev`.
 - `run`: loads a YAML config, discovers source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary.
+- `serve`: loads a YAML config, resolves HTTP upload bearer tokens, and runs the HTTP upload API.
 - `validate`: validates a local source bundle directory, a local source bundle tree, or one configured pipeline source.
 - `inspect`: validates source bundles and prints normalized bundle metadata for a local path or one configured pipeline source.
 - `manifest create`: creates `manifest.json` for a local source bundle directory.
 
-`validate` and `inspect` have two mutually exclusive modes: a local path shortcut, or configured source mode with `--config <path> --pipeline <id>`. Configured source mode opens only the selected pipeline source and supports configured `local`, `ssh`, and `s3` sources. It does not open destinations. `run` executes configured sources and destinations.
+`validate` and `inspect` have two mutually exclusive modes: a local path shortcut, or configured source mode with `--config <path> --pipeline <id>`. Configured source mode opens only the selected pipeline source and supports configured `local`, `ssh`, and `s3` sources. It does not open destinations. `run` executes configured `local`, `ssh`, and `s3` sources and destinations. `serve` executes configured `http_upload` sources through the upload API and normal destination fan-out.
 
 ## Flag reference
 
@@ -49,6 +51,10 @@ Output-producing subcommands:
 - `--dry-run`: load config, discover bundles, inspect destination state, print planned actions and final status, and do not write output files, destination state, or SSH `known_hosts` entries.
 - `--force`: allow explicit destructive replacement for supported conflict cases in this run only.
 
+`serve` flags:
+
+- `--config <path>`: config file to load. If omitted, `serve` uses `/usr/local/etc/distributor/config.yml`.
+
 `validate` and `inspect` configured source flags:
 
 - `--config <path>`: config file to load for source validation or inspection. Required in configured source mode.
@@ -127,6 +133,40 @@ Publish the local HTML example:
 go run ./cmd/distributor run --config examples/local-html.yml
 ```
 
+Start the HTTP upload API:
+
+```sh
+DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN=<token> \
+  go run ./cmd/distributor serve --config examples/http-upload-local.yml
+```
+
+Upload an archive to the configured `http_upload` pipeline associated with a bearer token:
+
+```sh
+curl -X POST http://127.0.0.1:8080/upload \
+  -H "Authorization: Bearer $DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN" \
+  -H "Content-Type: application/gzip" \
+  --data-binary @bundle.tar.gz
+```
+
+The upload response is accepted asynchronously:
+
+```json
+{"run_id":"reports.20260603T120000Z.abcdef12","status":"accepted"}
+```
+
+Check upload status:
+
+```sh
+curl http://127.0.0.1:8080/runs/<run-id>
+```
+
+Check server readiness:
+
+```sh
+curl http://127.0.0.1:8080/healthz
+```
+
 Preview local fan-out publication:
 
 ```sh

docs/config.md

@@ -2,15 +2,18 @@
 
 ## Config File Location
 
-`distributor run --config <path>` loads the YAML config at the provided path.
+`distributor run --config <path>` and `distributor serve --config <path>` load
+the YAML config at the provided path.
 
-If `--config` is omitted, `run` uses:
+If `--config` is omitted, both commands use:
 
 ```text
 /usr/local/etc/distributor/config.yml
 ```
 
-Config parsing rejects unknown YAML fields. The executable backends are `local`, `ssh`, and `s3`.
+Config parsing rejects unknown YAML fields. The executable `run` backends are
+`local`, `ssh`, and `s3`. The `serve` command executes `http_upload` sources
+through the HTTP upload API and normal destination fan-out.
 
 ## Minimal Local Config
 
@@ -31,6 +34,14 @@ This publishes source files only. It uses the default validation and transfer po
 ## Production-Oriented Local Config
 
 ```yaml
+server:
+  http:
+    bind: 127.0.0.1:8080
+    staging_root: /var/spool/distributor
+    max_upload_size: 20MB
+    queue_size: 16
+    max_concurrency: 1
+    retention: 24h
 pipelines:
   - id: reports
     source:
@@ -52,6 +63,84 @@ pipelines:
           on_conflict: fail
 ```
 
+## HTTP Upload Source Configuration
+
+HTTP upload sources are configured as pipeline sources only. They are not valid
+destination backends. `distributor serve` maps each configured upload token to
+exactly one `http_upload` pipeline.
+
+```yaml
+server:
+  http:
+    bind: 127.0.0.1:8080
+    staging_root: /var/spool/distributor
+    max_upload_size: 20MB
+    queue_size: 16
+    max_concurrency: 1
+    retention: 24h
+pipelines:
+  - id: weather-daily
+    source:
+      backend: http_upload
+      token_env: WEATHER_DAILY_UPLOAD_TOKEN
+      staging_path: /var/spool/distributor/weather-daily
+      max_upload_size: 20MB
+    destinations:
+      - id: archive
+        backend: local
+        path: /srv/reports/archive
+```
+
+`source.token_env` is required and names the environment variable or `secrets.directory` file that provides the bearer token. Literal upload tokens are not supported in YAML.
+
+`source.staging_path` is optional. When omitted, it defaults to `<server.http.staging_root>/<pipeline id>`.
+
+`source.max_upload_size` is optional. When omitted, it defaults to `server.http.max_upload_size`.
+
+The server resolves each `token_env` through the real process environment and
+the configured `secrets.directory` resolver. Startup fails if any configured
+upload token is missing, empty, or resolves to the same value as another upload
+pipeline. Token values are not read from YAML and are not printed in API
+responses.
+
+## HTTP Upload API
+
+`distributor serve` binds to `server.http.bind`, which defaults to
+`127.0.0.1:8080`.
+
+Routes:
+
+- `GET /healthz`: returns readiness status after config and upload tokens load.
+- `POST /upload`: accepts one tar or tar.gz source bundle archive.
+- `GET /runs/<run_id>`: returns an in-memory upload status record, or `404` if the run id is unknown or expired.
+
+`POST /upload` authenticates with:
+
+```text
+Authorization: Bearer <token>
+```
+
+The token selects the configured `http_upload` pipeline. Producers do not send a
+pipeline id. Requests with a submitted `pipeline` or `pipeline_id` query value
+are rejected.
+
+Accepted upload content types:
+
+- `application/x-tar`
+- `application/gzip`
+- `application/x-gzip`
+
+Accepted uploads return:
+
+```json
+{"run_id":"<id>","status":"accepted"}
+```
+
+The run id can be queried through `GET /runs/<run_id>` while the status record
+is retained in memory. Completed records expire after `server.http.retention`;
+expiration also removes committed staged bundle directories for completed
+uploads.
+
 ## HTML Publication
 
 To publish generated sidecar HTML from Markdown files:
@@ -143,6 +232,12 @@ Output URLs are built from `links.base_url`, the destination bundle path, and th
 
 Top level:
 
+- `server.http.bind`: optional HTTP bind address; defaults to `127.0.0.1:8080`.
+- `server.http.staging_root`: optional root for default HTTP upload staging paths; defaults to `/var/spool/distributor`.
+- `server.http.max_upload_size`: optional default upload size limit; defaults to `20MB`.
+- `server.http.queue_size`: optional HTTP upload admission queue size; defaults to `16`.
+- `server.http.max_concurrency`: optional HTTP upload worker concurrency; defaults to `1`.
+- `server.http.retention`: optional completed upload retention duration; defaults to `24h`.
 - `secrets.directory`: optional credential secrets directory.
 - `pipelines`: required non-empty list.
 
@@ -170,6 +265,9 @@ Source backend:
 - `force_path_style`: optional for `s3`; defaults to `true`. Set `false` only for services that require virtual-host addressing.
 - `credentials.access_key_id_env`: optional S3 credential environment variable name.
 - `credentials.secret_access_key_env`: optional S3 credential environment variable name.
+- `token_env`: required for `http_upload`; names the token environment variable or secret-file name.
+- `staging_path`: optional for `http_upload`; defaults below `server.http.staging_root` using the pipeline id.
+- `max_upload_size`: optional for `http_upload`; defaults to `server.http.max_upload_size`.
 
 Destination:
 
@@ -187,6 +285,20 @@ Accepted backend names:
 - `local`: executable; requires `path`.
 - `ssh`: executable; requires `host` and `path`.
 - `s3`: executable; requires `endpoint` and `bucket`.
+- `http_upload`: source-only configuration; requires `token_env`.
+
+## Size And Duration Values
+
+Upload size fields use an integer plus one of the supported binary-size suffixes:
+
+- `B`
+- `KB`
+- `MB`
+- `GB`
+
+Suffix multipliers use powers of 1024. Size values must be greater than zero after defaults are applied.
+
+HTTP retention uses Go-style duration strings such as `24h`, `90m`, or `168h`. Retention must be greater than zero after defaults are applied.
 
 ## SSH Backend
 
@@ -266,6 +378,14 @@ Defaults are applied after YAML decoding and before validation:
 - SSH `host_key_policy: accept-new`
 - S3 `region: us-east-1`
 - S3 `force_path_style: true`
+- `server.http.bind: 127.0.0.1:8080`
+- `server.http.staging_root: /var/spool/distributor`
+- `server.http.max_upload_size: 20MB`
+- `server.http.queue_size: 16`
+- `server.http.max_concurrency: 1`
+- `server.http.retention: 24h`
+- `source.staging_path: /var/spool/distributor/<pipeline id>` for `http_upload`
+- `source.max_upload_size: server.http.max_upload_size` for `http_upload`
 - `transform.markdown_to_html.mode: sidecar` when a Markdown-to-HTML transform block is present and mode is omitted
 - `publish.source: true`
 - `publish.html: false`
@@ -294,6 +414,10 @@ S3 credentials may name environment variables:
 - `credentials.access_key_id_env`
 - `credentials.secret_access_key_env`
 
+HTTP upload tokens name one environment variable or secret-file name:
+
+- `source.token_env`
+
 ## Examples
 
 Maintained examples live under [examples](../examples/):
@@ -304,5 +428,6 @@ Maintained examples live under [examples](../examples/):
 - `local-index.yml`: runnable local `index.html` publication.
 - `fan-out.yml`: runnable local fan-out publication to source and HTML destinations.
 - `archive-and-latest.yml`: runnable local fan-out publication to an archive destination and a fixed latest destination.
+- `http-upload-local.yml`: local HTTP upload server example with a token environment variable reference.
 - `ssh-destination.yml`: environment-gated local-to-SSH publication example.
 - `s3-destination.yml`: environment-gated local-to-S3 publication example.

docs/internal/app.md

@@ -2,66 +2,232 @@
 
 ## Purpose
 
-`internal/app` owns top-level use cases for `run`, `validate`, and `inspect`. It wires configuration, storage backends, transforms, publish planning, execution, summaries, and notification handoff.
+`internal/app` owns the top-level application use cases. It coordinates
+configuration loading, secret resolution, backend construction, source bundle
+discovery, destination selection, publish planning, publish execution,
+notification handoff, run reporting, and in-memory run coordination.
 
-## Inputs and outputs
+The package is the boundary between callers and lower-level domain packages. It
+does not own manifest validation rules, destination state comparison, storage
+path rules, output planning, transform rendering, or backend-specific behavior.
 
-`Run` accepts a context, optional config path, dry-run flag, force flag, stdout writer, output format, and optional notifier. It loads YAML config, discovers source bundles for each configured pipeline, plans each destination independently, optionally executes publish plans, writes text or JSON output when stdout is supplied, and returns an aggregated error if any destination fails.
+## Use Cases
 
-`Validate` and `Inspect` accept either a local path or one configured pipeline source. `Validate` discovers and validates bundles. `Inspect` writes bundle metadata and manifest file entries to stdout when provided.
+`Run` is the CLI-facing all-pipeline entrypoint. It accepts a context, optional
+config path, dry-run flag, force flag, stdout writer, output format, and
+optional notifier. It runs every configured pipeline, builds a `RunReport`, and
+projects the report to text or JSON when stdout is supplied.
 
-## Run flow
+`RunPipeline` is the app-layer single-pipeline entrypoint. It accepts a context,
+config path, pipeline ID, dry-run flag, force flag, and optional notifier. It
+loads the same config as `Run`, narrows execution to exactly one configured
+pipeline, and returns a `RunReport` without writing command output.
 
-The runner:
+`RunPipelineWithLocalSource` is the app-layer single-pipeline entrypoint for an
+already prepared local source bundle root. It accepts the same pipeline
+selection and execution options as `RunPipeline` plus a local source root path.
+It loads config, selects one configured pipeline, opens the supplied source
+root as a local backend, validates exactly that root bundle, and then uses the
+same destination fan-out path as normal runs.
+
+`Validate` and `Inspect` accept either a local path or one configured pipeline
+source. They share source backend construction with run workflows and never open
+destination backends.
+
+`Serve` is the CLI-facing HTTP upload server entrypoint. It loads config,
+loads the configured secrets directory, resolves upload bearer tokens for
+configured `http_upload` sources, creates an `UploadCoordinator`, binds
+`server.http.bind`, and serves the upload API until its context is cancelled.
+
+## Run Reports
+
+`RunReport` is the structured result model for run workflows. It includes
+dry-run state, pipeline summaries, action records, output metadata, summary
+counters, warnings, and destination-scoped output errors.
+
+Text and JSON run output are projections of `RunReport`. JSON tags on report
+records match the CLI JSON output contract. Text output preserves the CLI
+summary shape while keeping output rendering outside the core planning and
+execution loop.
+
+Destination-scoped failures produce a report plus an aggregated error. Fatal
+setup failures, such as config loading, source open, or source discovery
+failures, return before a complete run report is available.
+
+## Run Flow
+
+The app runner:
 
 1. loads config from the supplied path or `config.DefaultConfigPath`;
-2. opens the configured source backend;
+2. loads configured secret files into a config-owned environment resolver;
-3. discovers validated bundles from the source root;
+3. builds the app-level backend factory and transform registry;
-4. selects source bundles for each destination according to destination path mapping;
+4. opens each selected pipeline source backend;
-5. opens each destination backend independently;
+5. discovers validated source bundles from the source root;
-6. builds publish plans for the selected bundle and destination combinations;
+6. selects source bundles for each destination according to path mapping;
-7. prints plan lines or JSON action records and records summary counters;
+7. opens destination backends independently;
-8. executes publish or replacement plans unless dry-run is enabled;
+8. builds publish plans for selected bundle and destination combinations;
-9. invokes the notifier after successful publish or replacement actions.
+9. records warnings, action records, output metadata, and summary counters;
+10. executes publish or replacement plans unless dry-run is enabled;
+11. invokes the notifier after successful publish or replacement actions;
+12. returns the structured report and any aggregated destination failures.
 
-Destination failures are collected while later destinations continue to run. Source open and source discovery failures stop the run because there are no valid bundles to fan out.
+`RunPipeline` follows the same flow after selecting a single configured
+pipeline. It uses the same backend factory, secret loading, transform registry,
+warning generation, destination planning, publish execution, notification
+behavior, and failure aggregation as `Run`.
 
-## Run implementation
+`RunPipelineWithLocalSource` follows the same flow after pipeline selection
+except for source opening and source discovery. It opens the supplied local
+source root directly, validates the root bundle before opening any destinations,
+and passes the resulting local source backend and bundle into the same
+destination planning and execution loop. Destination code receives the normal
+storage backend and bundle values and does not depend on how the source root was
+prepared.
 
-`run.go` contains the public `Run` entrypoint and the main configuration orchestration path. Package-local run helpers are grouped by responsibility:
+## Upload Coordination
 
-- `run_selection.go`: destination bundle selection, path mapping decisions, and fixed-path warnings;
+`UploadCoordinator` owns in-memory coordination for asynchronous upload
-- `run_warnings.go`: secret and SSH warning data;
+processing. It admits uploads for configured `http_upload` pipelines, generates
-- `run_output.go`: text plan lines, JSON action records, and output projections;
+run IDs, tracks status records, stages accepted archives through
-- `run_summary.go`: summary counters and JSON summary records;
+`internal/ingest`, and executes the selected pipeline through
-- `run_failures.go`: destination failure aggregation and partial-result detection;
+`RunPipelineWithLocalSource`.
+
+Upload run IDs use:
+
+```text
+<pipeline id>.<UTC timestamp>.<random suffix>
+```
+
+The timestamp uses `YYYYMMDDThhmmssZ` UTC format and the suffix is filesystem
+safe.
+
+The coordinator records these statuses:
+
+- `accepted`
+- `queued`
+- `running`
+- `succeeded`
+- `failed`
+- `expired`
+
+Admission is bounded by `server.http.queue_size`. Full queues are rejected
+before the upload body is staged. Execution is bounded by
+`server.http.max_concurrency`, and only one upload for a given pipeline may run
+at a time. Later uploads for the same pipeline remain queued until the active
+run finishes.
+
+Completed records retain the final run report or error text until
+`server.http.retention` elapses. Expiration removes completed status records and
+their committed staged bundle directories. The coordinator is memory-only and
+does not persist queue state, status records, or run reports.
+
+## HTTP Upload Server
+
+The HTTP upload server is app-layer transport wiring around
+`UploadCoordinator`. It owns request authentication, route dispatch, HTTP status
+mapping, and JSON response projection. Bundle staging and publication remain in
+the coordinator and staged-source run path.
+
+Server startup resolves every configured `http_upload` source `token_env`
+through the config-owned environment resolver after `secrets.directory` has
+been loaded. Startup fails when a token is missing, empty, or duplicates another
+upload pipeline token. Error messages identify environment variable names and
+pipeline ids, but not token values.
+
+Routes:
+
+- `GET /healthz`: returns `200` after config, secrets, tokens, coordinator, and route setup succeed.
+- `POST /upload`: accepts authenticated tar and tar.gz archives and returns an accepted run id.
+- `GET /runs/<run_id>`: returns the current in-memory upload status record or `404`.
+
+The upload token maps to exactly one configured pipeline. Producers do not
+submit pipeline ids, and submitted `pipeline` or `pipeline_id` query values are
+rejected. Full queues are rejected before the request body is read. Oversized
+uploads, unsupported content types, invalid bearer tokens, full queues, and
+unknown status records are mapped to stable HTTP status codes without returning
+secret token values.
+
+## Coordination
+
+`PipelineRunCoordinator` wraps `RunPipeline` with in-memory admission control.
+It allows different pipeline IDs to run concurrently and rejects a second active
+run for the same pipeline ID.
+
+Coordinator records contain a run ID, pipeline ID, status, timestamps, completed
+report, and error text when applicable. Active state is memory-only and is
+cleared after success, failure, unknown pipeline ID, or context cancellation.
+
+The admission context is checked before a run is accepted. Once accepted, the
+run uses the coordinator lifetime context, so caller cancellation can stop
+waiting for admission without owning the actual run lifetime.
+
+The coordinator does not queue duplicate runs, persist run records, or define
+transport endpoints.
+
+## Errors
+
+`Run` returns immediately for config loading errors, context cancellation before
+work starts, source open errors, and source discovery errors.
+
+`RunPipeline` returns `PipelineNotFoundError` when the requested pipeline ID is
+not configured. Callers can detect that condition with `IsPipelineNotFound`.
+
+`RunPipelineWithLocalSource` also returns `PipelineNotFoundError` for an unknown
+pipeline ID. It returns before destination opening when the supplied local
+source root is missing, cannot be opened, or does not validate as one complete
+source bundle.
+
+Per-destination backend, planning, execution, and notification errors are
+aggregated into one run error after remaining destinations have been attempted.
+Destination diagnostics include pipeline ID, destination ID, backend, and
+bundle path.
+
+`PipelineRunCoordinator` returns `DuplicatePipelineRunError` when the same
+pipeline already has an active run. Callers can detect that condition with
+`IsDuplicatePipelineRun`.
+
+Stdout write errors are returned immediately because the caller's requested
+output stream can no longer be trusted.
+
+## Package Layout
+
+Run helpers are grouped by responsibility:
+
+- `run.go`: `Run`, `RunPipeline`, and shared run orchestration.
+- `run_output.go`: `RunReport`, action/output records, and text/JSON report projection.
+- `run_summary.go`: summary counters.
+- `run_failures.go`: destination failure aggregation and partial-result detection.
+- `run_selection.go`: destination bundle selection, path mapping decisions, and fixed-path warnings.
+- `run_warnings.go`: secret and SSH warning records.
 - `run_notify.go`: notification event projection and action filtering.
+- `run_coordinator.go`: in-memory run admission, run IDs, status records, and duplicate-run errors.
+- `upload_coordinator.go`: in-memory upload admission, queueing, status tracking, staging handoff, and staged-source execution.
+- `upload_http.go`: HTTP upload authentication, routes, JSON response projection, and HTTP error mapping.
+- `serve.go`: config/secrets loading and HTTP server startup.
+- `backends.go`: app-level backend factory wiring.
+- `transforms.go`: app-level transform registry wiring.
+- `source_select.go`: configured-source selection shared by `validate` and `inspect`.
 
-These helpers remain in `internal/app` because command output, warning collection, destination failure aggregation, notifier handoff, and backend construction are app-owned orchestration concerns.
+## Backend And Transform Wiring
 
-## Backend and transform wiring
+The app-level backend factory registers local, SSH, and S3 backends for runtime
+execution. Source and destination backend config is converted through a shared
+app-local open spec before adapter construction.
 
-The app-level backend factory registers local, SSH, and S3 backends for execution. Source and destination backend config is converted through a shared app-local open spec before adapter construction. S3 explicit credential references are resolved through the config environment resolver.
+Credential references are resolved through the config environment resolver.
+Production app code must not read backend credential environment variables
+directly.
 
-The app-level transform registry registers Markdown-to-HTML using `internal/transform/markdown`. Lower-level publish code receives a resolver and does not import concrete transform implementations.
+The app-level transform registry registers Markdown-to-HTML through
+`internal/transform/markdown`. Lower-level publish code receives a resolver and
+does not import concrete transform implementations.
 
-## Dry-run behavior
+## Dry-Run Behavior
 
-Dry-run still loads config, opens backends, discovers bundles, inspects destinations, resolves transforms, and builds publish plans. It does not write destination outputs, write `.distributor.json`, delete managed outputs, perform forced prefix deletion, or notify.
+Dry-run loads config, opens backends, discovers bundles, inspects destinations,
-
+resolves transforms, and builds publish plans. It does not write destination
-## Failure behavior
+outputs, write `.distributor.json`, delete managed outputs, perform forced
-
+prefix deletion, or invoke notifications.
-`Run` returns immediately for config loading errors, context cancellation before work starts, source open errors, and source discovery errors. Per-destination backend, planning, execution, and notification errors are aggregated into one run error after remaining destinations have been attempted.
-
-Run diagnostics include pipeline id, destination id, destination backend, and bundle path for destination-scoped failures. Source open and discovery failures include the source backend.
-
-Stdout write errors are returned immediately because the caller's requested output stream can no longer be trusted.
-
-## Boundaries
-
-`internal/app` coordinates packages but does not own manifest validation rules, destination state comparison, storage path rules, output planning, transform rendering, or backend-specific filesystem behavior.
-
-Configured-source `Validate` and `Inspect` share source backend construction with `Run` and do not open destinations.
 
 ## Tests
 
@@ -71,10 +237,18 @@ Before changing app orchestration, inspect tests under:
 - `internal/cli`
 - `internal/publish`
 
+Use focused app tests for report structure, single-pipeline execution,
+coordinator admission, warning generation, notification behavior, and
+partial-result aggregation.
+
 ## Invariants
 
 - One source fans out to each destination independently.
 - Destination failures do not prevent later destinations from being planned.
+- Destination-scoped failures still produce a structured report plus an aggregated error.
 - Dry-run must not mutate destination storage or invoke notifications.
+- `RunPipeline` must use the same run path as `Run` after pipeline selection.
+- Duplicate in-flight runs are rejected only for the same pipeline ID.
+- Different pipeline IDs may run concurrently.
 - Concrete backend and transform registration stays at the app layer.
 - The default notifier is `notify.Noop`.

docs/internal/config.md

@@ -6,7 +6,7 @@
 
 ## Inputs and outputs
 
-Input is a YAML file containing optional `secrets` and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
+Input is a YAML file containing optional `server`, optional `secrets`, and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
 
 ## Loading flow
 
@@ -14,12 +14,21 @@ Input is a YAML file containing optional `secrets` and required `pipelines`. Out
 
 Known-field checking rejects misspelled or unknown YAML keys before defaults and validation run.
 
-`LoadFile` does not read secret files. `Run` loads the configured secrets directory after config validation and before backend construction.
+`LoadFile` does not read secret files. App entrypoints load the configured
+secrets directory after config validation and before credential-consuming work.
 
 ## Defaults
 
 Defaults are applied in `ApplyDefaults`:
 
+- HTTP server `bind` defaults to `127.0.0.1:8080`;
+- HTTP server `staging_root` defaults to `/var/spool/distributor`;
+- HTTP server `max_upload_size` defaults to `20MB`;
+- HTTP server `queue_size` defaults to `16`;
+- HTTP server `max_concurrency` defaults to `1`;
+- HTTP server `retention` defaults to `24h`;
+- `http_upload` source `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`;
+- `http_upload` source `max_upload_size` defaults to `server.http.max_upload_size`;
 - pipeline validation defaults `on_digest_mismatch` to `fail`;
 - SSH backend `port` defaults to `22`;
 - SSH backend `host_key_policy` defaults to `accept-new`;
@@ -34,7 +43,11 @@ Defaults are applied in `ApplyDefaults`:
 
 ## Validation responsibilities
 
-Validation requires at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions.
+Validation requires positive HTTP server limits and retention, at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions.
+
+HTTP upload sources require `token_env`, a staging path after defaults, and a positive maximum upload size. Literal token fields are not part of the YAML schema. The `http_upload` backend is accepted only for sources and rejected for destinations.
+
+Upload size values are parsed from strings with `B`, `KB`, `MB`, or `GB` suffixes using 1024 multipliers. Retention values are parsed with `time.ParseDuration`. Explicit zero values fail validation; omitted values receive defaults before validation.
 
 Transfer validation accepts `replace` for `on_destination_newer` and `on_conflict`, but publish planning honors those destructive actions only when the current run explicitly requests force.
 
@@ -46,12 +59,14 @@ Destination links are optional. When a `links` block is present, `base_url` is r
 
 ## Executable support boundary
 
-Config validation accepts `local`, `ssh`, and `s3` backend shapes. Runtime execution opens all three through `internal/app`.
+Config validation accepts `local`, `ssh`, `s3`, and source-only `http_upload` backend shapes. Runtime `run`, `validate`, and `inspect` workflows open `local`, `ssh`, and `s3` through `internal/app`. Runtime `serve` workflows execute `http_upload` sources through the app upload coordinator and HTTP server.
 
 SSH config uses structured fields: `host`, optional `user`, optional `port`, `path`, optional `ssh_key_file`, optional `known_hosts`, and optional `host_key_policy`. `host_key_policy` accepts YAML booleans and strings and normalizes `true`/`strict`, `accept-new`, and `false`/`off`.
 
 S3 config requires `endpoint` and `bucket`, normalizes optional `prefix`, defaults `region` to `us-east-1`, and defaults omitted `force_path_style` to `true` while preserving explicit `false`.
 
+HTTP upload config is source-only. Config owns its YAML shape, defaulting, size and duration parsing, and validation. The config package does not authenticate requests, stage uploads, or execute HTTP upload sources. The app layer resolves `token_env` through the config-owned environment resolver before starting the HTTP server.
+
 ## Secrets and credential resolution
 
 `secrets.directory` points to a directory of credential files. `LoadSecretEnvironment` reads regular files and symlinks to regular files, rejects invalid filenames, trims exactly one trailing LF or CRLF, and returns an `Environment` resolver plus conflict metadata.

docs/internal/ingest.md

@@ -0,0 +1,43 @@
+# Ingestion Internals
+
+## Purpose
+
+`internal/ingest` stages uploaded source bundle archives into local per-run directories. It does not authenticate requests, manage upload queues, publish destinations, or start an HTTP server.
+
+## Archive staging
+
+`StageArchive` accepts one upload body, content type, pipeline staging path, run id, and explicit size and file-count limits. It writes the request body to temporary storage while enforcing the configured upload size limit, extracts the archive into temporary local storage, validates the extracted source bundle, and then commits the validated bundle to:
+
+```text
+<pipeline staging path>/<run id>
+```
+
+The returned `StagedBundle.Root` is a local filesystem path to the validated source bundle root.
+
+## Accepted archive formats
+
+The package accepts only:
+
+- `application/x-tar`
+- `application/gzip`
+- `application/x-gzip`
+
+Gzip uploads must contain a tar archive.
+
+## Extraction rules
+
+Archive entry paths must be clean relative slash-separated paths. Extraction rejects absolute paths, path traversal, backslash paths, duplicate files, symlinks, hardlinks, devices, sockets, and other special entries.
+
+The archive must contain exactly one root-level `manifest.json`. Nested manifests are rejected.
+
+Regular files and directories are the only accepted tar entries. Regular file extraction enforces the explicit maximum extracted byte count and maximum file count supplied by the caller.
+
+## Bundle validation
+
+After extraction, the package loads and validates the staged bundle through `pkg/bundle`. Manifest parsing, source path validation, file existence checks, regular-file checks, file sizes, file SHA-256 digests, and bundle digest validation use the existing source bundle contract.
+
+Validation happens before the staged bundle is committed to its final per-run path.
+
+## Failure behavior
+
+Failed staging removes temporary archive and extraction data created by the package. A failed call does not publish anything and does not leave a committed per-run bundle directory.

docs/operations.md

@@ -68,6 +68,84 @@ Validate one configured source without opening destinations:
 go run ./cmd/distributor validate --config examples/local-publish.yml --pipeline example-source-bundle
 ```
 
+## HTTP Upload Workflow
+
+`distributor serve` runs the HTTP upload API for pipelines whose source backend
+is `http_upload`. Each upload token maps to one configured pipeline, and each
+accepted archive is staged, validated, and published through the same
+destination fan-out path used by local source runs.
+
+Minimal local HTTP upload configuration:
+
+```yaml
+server:
+  http:
+    bind: 127.0.0.1:8080
+    staging_root: /var/spool/distributor
+    max_upload_size: 20MB
+    queue_size: 16
+    max_concurrency: 1
+    retention: 24h
+secrets:
+  directory: /run/secrets/distributor
+pipelines:
+  - id: reports
+    source:
+      backend: http_upload
+      token_env: DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN
+    destinations:
+      - id: archive
+        backend: local
+        path: /srv/reports/archive
+```
+
+Create `/run/secrets/distributor/DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` or set the
+real process environment variable before starting the server. Distributor does
+not read literal upload tokens from YAML.
+
+Start the maintained local example:
+
+```sh
+DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN=<token> \
+  go run ./cmd/distributor serve --config examples/http-upload-local.yml
+```
+
+Submit a tar or tar.gz source bundle:
+
+```sh
+curl -X POST http://127.0.0.1:8080/upload \
+  -H "Authorization: Bearer $DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN" \
+  -H "Content-Type: application/gzip" \
+  --data-binary @bundle.tar.gz
+```
+
+Successful admission returns a run id:
+
+```json
+{"run_id":"reports.20260603T120000Z.abcdef12","status":"accepted"}
+```
+
+Poll status until it reaches `succeeded` or `failed`:
+
+```sh
+curl http://127.0.0.1:8080/runs/<run-id>
+```
+
+The status record includes the completed run report on successful publication
+or error details on failure. Status is memory-only and expires after
+`server.http.retention`; completed staged bundle directories are removed on
+expiry. Restarting the process clears upload status and queue state.
+
+Use `GET /healthz` for readiness after config and tokens load:
+
+```sh
+curl http://127.0.0.1:8080/healthz
+```
+
+The default bind address is private loopback. Put TLS, public routing,
+rate-limiting, and external access policy in a reverse proxy or deployment
+layer.
+
 ## Filesystem Layout
 
 Source bundles are discovered beneath the configured source root. Each bundle is a directory containing `manifest.json`.
@@ -322,7 +400,10 @@ secrets:
   directory: /run/secrets/distributor
 ```
 
-The directory is loaded during `run` and configured-source `validate` or `inspect` before any backend is opened. If the directory is missing, unreadable, or contains an invalid secret filename, the command fails before storage work starts.
+The directory is loaded during `run`, `serve`, and configured-source `validate`
+or `inspect` before credential-consuming work starts. If the directory is
+missing, unreadable, or contains an invalid secret filename, the command fails
+before storage work starts.
 
 Real process environment values take precedence over files with the same name. If the values differ and stdout is enabled, `run` and configured-source diagnostics print a warning naming the ignored secret file variable without printing either value. The process environment is not changed.
 

docs/roadmap/audit.md

@@ -0,0 +1,592 @@
+# Code Quality and Deduplication Audit
+
+## 1. Executive summary
+
+Overall code quality is strong. The repository has clear package boundaries, good current-behavior documentation, focused adapter packages, and tests close to most implemented behavior. The most important cleanup opportunities are narrow and behavior-preserving rather than architectural.
+
+Top three refactoring targets:
+
+1. HTTP upload admission, body staging, and archive validation are split across `internal/app` and `internal/ingest` in a way that duplicates size and content-type policy and buffers uploads in memory.
+2. Runtime config loading, default config path selection, secret loading, and warning projection are repeated across app entrypoints.
+3. Run orchestration mixes destination processing, failure aggregation, warning recording, and report event ordering in one large loop, making future changes harder to review safely.
+
+The codebase appears ready for a limited cleanup pass. I do not see a major architectural risk that requires a redesign before the next release.
+
+## 2. Repository map reviewed
+
+Reviewed policy and current-behavior documentation:
+
+- `AGENTS.md`
+- `README.md`
+- `docs/policy/architecture.md`
+- `docs/policy/development.md`
+- `docs/policy/documentation.md`
+- `docs/config.md`
+- `docs/cli.md`
+- `docs/operations.md`
+- `docs/troubleshooting.md`
+- `docs/internal/*.md`
+- `docs/roadmap/http.md`
+- `docs/roadmap/implementation.md`
+
+Reviewed implementation areas:
+
+- `cmd/distributor`: executable entrypoint.
+- `internal/cli`: root command, `version`, `run`, `serve`, `validate`, `inspect`, and `manifest create` parsing.
+- `internal/app`: run orchestration, configured source diagnostics, backend factory, manifest creation, CLI output, HTTP upload server, upload coordinator, and pipeline coordinator.
+- `internal/config`: config structs, defaults, validation, quantity parsing, S3/SSH helpers, and secrets resolver.
+- `internal/bundle`: storage-backed source discovery and validation.
+- `pkg/bundle`: public manifest model, digest logic, manifest building, local validation, and local bundle writer.
+- `internal/storage` and `internal/storage/fake`: backend interface, path helpers, walk helpers, typed errors, and fake backend.
+- `internal/adapters/local`, `internal/adapters/ssh`, and `internal/adapters/s3`: runtime storage adapters.
+- `internal/ingest`: HTTP upload archive staging.
+- `internal/publish`: destination planning, output selection, link projection, state writing, cleanup, and force replacement.
+- `internal/state`: destination state parsing, validation, comparison, and JSON projection.
+- `internal/transform` and `internal/transform/markdown`: transform registry and Markdown rendering.
+- `internal/link`, `internal/notify`, `internal/logging`, and `internal/testutil`.
+- `examples`, package tests, and package `testdata`.
+
+Requested areas that are absent as separate packages:
+
+- `internal/stage`
+- `internal/modules`
+- `internal/validators`
+- `internal/artifacts`
+- `internal/manifest`
+- `internal/schema`
+- `internal/report`
+- public `pkg` packages other than `pkg/bundle`
+
+Those absences are consistent with current architecture policy; the corresponding behavior lives in narrower existing packages.
+
+## 3. High-confidence deduplication opportunities
+
+### HTTP upload body handling should be owned by ingestion
+
+Affected files/packages:
+
+- `internal/app/upload_http.go`
+- `internal/app/upload_coordinator.go`
+- `internal/ingest/archive.go`
+- `internal/app/upload_http_test.go`
+- `internal/app/upload_http_integration_test.go`
+- `internal/ingest/archive_test.go`
+
+Duplicated or near-duplicated behavior:
+
+- `internal/app/upload_http.go` validates upload content types in `supportedUploadContentType`, while `internal/ingest/archive.go` validates the same content types in `archiveFormat`.
+- `internal/app/upload_http.go` enforces upload size in `readUploadBody`, while `internal/ingest/archive.go` enforces upload size again in `writeLimited`.
+- The HTTP handler reads the full upload body into memory before submission, then the coordinator passes a `bytes.Reader` to ingestion.
+
+Why it matters:
+
+- The app transport layer now partially owns archive policy that should belong to `internal/ingest`.
+- Large accepted uploads are buffered in memory even though ingestion already has streaming-to-disk mechanics.
+- Future archive formats, content types, or upload limit changes would need coordinated edits in multiple packages.
+
+Recommended refactor:
+
+- Move supported content-type checking behind an ingestion-owned helper, for example `ingest.ValidateContentType` or `ingest.IsSupportedContentType`.
+- Change upload admission so the request body is streamed to staging exactly once before the HTTP handler returns `202 Accepted`.
+- Keep queue-full rejection before reading the body.
+- Queue a staged local bundle root, not an unread request body. This preserves async distribution while keeping HTTP request lifetime separate from later pipeline execution.
+- Keep `UploadCoordinator` responsible for queueing, status, per-pipeline serialization, and execution. Keep `internal/ingest` responsible for archive format, size, extraction, cleanup, and source bundle validation.
+
+Suggested tests:
+
+- HTTP handler rejects full queues without reading the body.
+- HTTP handler streams a valid body to ingestion and returns `202` only after staging succeeds.
+- Unsupported content types are rejected through the ingestion-owned content-type policy.
+- Oversized uploads are rejected without retaining a staged run.
+- Accepted upload status still transitions through queued/running/succeeded or failed without depending on an open HTTP request body.
+
+Risk level:
+
+- Medium. The behavior change is internal but touches admission timing and async execution boundaries. It should be implemented in a focused prompt with existing HTTP integration tests extended first.
+
+### Runtime config and secret setup should have one app-level helper
+
+Affected files/packages:
+
+- `internal/app/run.go`
+- `internal/app/source_select.go`
+- `internal/app/serve.go`
+- `internal/app/backends.go`
+- `internal/app/run_warnings.go`
+- `internal/config`
+- `internal/app/*_test.go`
+
+Duplicated or near-duplicated behavior:
+
+- Defaulting an empty config path to `config.DefaultConfigPath` appears in `Run`, `RunPipeline`, `RunPipelineWithLocalSource`, and `Serve`.
+- Config loading and secret loading are separate repeated steps in `buildRunReport`, `selectSourceBundlesFromConfig`, and `Serve`.
+- Secret conflict warnings are projected in run and configured source diagnostics, while serve loads secrets without using or exposing conflict warning metadata.
+- Backend factory construction from a config environment is repeated through provider plumbing.
+
+Why it matters:
+
+- Config and secret precedence is a public operational policy.
+- A future change to config discovery, secret conflict reporting, or runtime environment construction could drift between `run`, `serve`, `validate`, and `inspect`.
+- Tests for secrets and credential resolution need to cover several entrypoints today.
+
+Recommended refactor:
+
+- Add a small app-level runtime setup helper, for example `loadRuntimeConfig(optionsConfigPath string) (runtimeConfig, error)`.
+- The helper should own default config path selection, `config.LoadFile`, `config.LoadSecretEnvironment`, and conversion of secret conflicts into `OutputWarning` values.
+- Keep config parsing and validation in `internal/config`; the helper should not duplicate config policy.
+- Let `run`, configured `validate`/`inspect`, and `serve` call the helper and then apply command-specific behavior.
+
+Suggested tests:
+
+- One focused app test proving default config path selection remains unchanged where injection permits it.
+- Existing secret conflict JSON/text warning tests for `run`, `validate`, and `inspect`.
+- Serve startup test proving duplicate and missing upload tokens still fail without leaking values.
+- S3 explicit credential tests proving the resolver is still used through the helper.
+
+Risk level:
+
+- Low to medium. This is a straightforward centralization but touches several command entrypoints.
+
+### Run destination processing needs a narrow helper boundary
+
+Affected files/packages:
+
+- `internal/app/run.go`
+- `internal/app/run_output.go`
+- `internal/app/run_failures.go`
+- `internal/app/run_selection.go`
+- `internal/publish`
+- `internal/app/run_test.go`
+
+Duplicated or near-duplicated behavior:
+
+- Destination backend open failures and publish planning/execution failures each manually add `runFailures`, record summary failure counts, append `RunActionRecord`, and append pipeline event indexes.
+- `publish.Build` error handling patches missing `Plan` identity fields inline before converting the plan to a run action.
+- Fixed-path warning emission is interleaved with destination selection and publish plan handling.
+
+Why it matters:
+
+- `run --format json` depends on exact action ordering, warnings, partial failures, and summary counters.
+- Future changes to actions, links, notifications, or HTTP upload reports could accidentally update one failure path but not another.
+- The current loop is correct but dense enough that small behavior changes are hard to review.
+
+Recommended refactor:
+
+- Extract a narrow `runDestination` or `destinationRunner` helper that processes one destination and returns action records, warnings, summary deltas, and failures.
+- Add a helper for recording a destination-scoped failure that updates `runFailures`, `runSummary`, `RunReport.Actions`, and pipeline events in one place.
+- Add a helper that normalizes partial `publish.Plan` identity fields before action projection.
+- Do not introduce a generic workflow engine or stage abstraction.
+
+Suggested tests:
+
+- Preserve existing run text output golden assertions.
+- Preserve JSON partial-result behavior when planning fails after destination processing begins.
+- Add one focused test where destination open fails for multiple selected bundles and verify action records, output errors, and summary counters stay aligned.
+- Add one fixed-path dry-run warning test after extraction to verify event ordering.
+
+Risk level:
+
+- Medium. The refactor is behavior-preserving but touches the most important user-facing workflow.
+
+### Archive path validation duplicates source path policy with a different error surface
+
+Affected files/packages:
+
+- `internal/ingest/archive.go`
+- `pkg/bundle/path.go`
+- `internal/storage/path.go`
+- `internal/ingest/archive_test.go`
+
+Duplicated or near-duplicated behavior:
+
+- `cleanArchivePath`, `pkg/bundle.ValidateSourcePath`, and `storage.ValidatePath` all enforce clean slash-separated relative paths with no backslashes, no absolute paths, and no dot segments.
+- Archive staging needs slightly different policy because directories are allowed and `manifest.json` is allowed only at the root, so the duplication is not completely mechanical.
+
+Why it matters:
+
+- Path safety is high-risk behavior.
+- Future changes to source path rules could miss archive extraction, especially around backslashes, reserved names, or dot segments.
+
+Recommended refactor:
+
+- Keep archive-specific rules in `internal/ingest`, but use a shared path-checking primitive where possible.
+- A good shape is an exported `pkg/bundle.ValidatePathSegmented` only if it fits the public producer API, or an internal helper in ingestion that delegates file-entry validation to `pkg/bundle.ValidateSourcePath` for regular files after handling directory-specific exceptions.
+- Preserve current archive-specific errors and tests.
+
+Suggested tests:
+
+- Table tests shared or mirrored across bundle path validation and archive path cleaning for absolute paths, traversal, backslashes, dot segments, empty names, root `manifest.json`, nested `manifest.json`, and `.distributor.json`.
+- Regression tests proving directories are still accepted in archives but symlinks and hardlinks remain rejected.
+
+Risk level:
+
+- Low to medium. Path validation changes need careful tests, but the desired change can be small.
+
+## 4. Medium-confidence opportunities
+
+### Source and destination backend config shapes could expose a normalized view
+
+Affected files/packages:
+
+- `internal/config/config.go`
+- `internal/config/defaults.go`
+- `internal/config/validate.go`
+- `internal/app/backends.go`
+- `internal/config/*_test.go`
+- `internal/app/backends_test.go`
+
+Duplicated or near-duplicated behavior:
+
+- `config.Backend` and `config.Destination` duplicate backend fields for local, SSH, S3, and credentials.
+- Defaults for source backends and destination backends are implemented in separate functions.
+- App backend opening converts both shapes into `backendOpenSpec`.
+
+Why it matters:
+
+- New backend fields must be added to both YAML structs, defaulting paths, validation paths, app open-spec conversion, docs, and tests.
+- The current pattern is easy to understand but likely to drift as more backend-specific fields are added.
+
+Recommended refactor:
+
+- Keep the YAML shape unchanged for compatibility.
+- Add package-local helpers in `internal/config` that return a normalized backend view for either source or destination.
+- Use that view for shared backend defaulting and validation where it improves clarity.
+- Keep destination-only fields such as `publish`, `transfer`, `links`, and `path_mapping` on `Destination`.
+
+Suggested tests:
+
+- Existing source and destination backend validation tests should continue to pass.
+- Add a table test that validates equivalent local, SSH, and S3 source/destination backend field requirements through the shared view.
+- Add a test that `http_upload` remains source-only.
+
+Risk level:
+
+- Medium. This reduces future drift, but the current duplication is understandable and does not need to be the first cleanup.
+
+### CLI command scaffolding is mostly shared, but manifest create has special parsing
+
+Affected files/packages:
+
+- `internal/cli/run.go`
+- `internal/cli/serve.go`
+- `internal/cli/source_mode.go`
+- `internal/cli/manifest.go`
+- `internal/cli/version.go`
+- `internal/cli/root_test.go`
+
+Duplicated or near-duplicated behavior:
+
+- Several commands repeat `flag.NewFlagSet`, `SetOutput`, help handling, format parsing, and usage exit handling.
+- `manifest create` uses `splitManifestCreateArgs` to allow a positional bundle path before flags, unlike Go's default `flag` behavior.
+
+Why it matters:
+
+- CLI syntax and error behavior are public.
+- A broad CLI helper could accidentally obscure command-specific parsing, but a narrow helper could reduce repeated setup and invalid-format handling.
+
+Recommended refactor:
+
+- Do not introduce a CLI framework.
+- Consider a tiny helper for common `FlagSet` creation and output-format parsing after higher-value app/config cleanup.
+- Keep `manifest create` custom parsing local unless another command needs the same interspersed positional behavior.
+
+Suggested tests:
+
+- Preserve current CLI usage-error tests.
+- Add explicit tests for `manifest create <path> --id x`, `manifest create --id x <path>`, and invalid missing flag values before any parser cleanup.
+
+Risk level:
+
+- Low if kept narrow; medium if over-generalized.
+
+### Output DTOs repeat bundle metadata projection
+
+Affected files/packages:
+
+- `internal/app/validate.go`
+- `internal/app/inspect.go`
+- `internal/app/manifest.go`
+- `internal/app/run_output.go`
+- `internal/app/output.go`
+
+Duplicated or near-duplicated behavior:
+
+- `inspect` and `manifest create` both project bundle file metadata into command-specific JSON structs.
+- `validate`, `inspect`, and `manifest create` each define local result types and file record types.
+- RFC3339 formatting uses both `time.RFC3339` and the equivalent literal layout string.
+
+Why it matters:
+
+- JSON output is now a public interface.
+- Repeated projection can drift in field names, timestamp formatting, or path display rules.
+
+Recommended refactor:
+
+- Add a small app-local projection helper for bundle summaries and manifest file records.
+- Use `time.RFC3339` instead of literal RFC3339 layouts.
+- Keep command-specific result structs where the command output semantics differ.
+
+Suggested tests:
+
+- JSON structural tests for `validate`, `inspect`, and `manifest create` before and after the helper.
+- A timestamp-format assertion using an offset timestamp to confirm current behavior is preserved.
+
+Risk level:
+
+- Low.
+
+### PipelineRunCoordinator overlaps conceptually with UploadCoordinator
+
+Affected files/packages:
+
+- `internal/app/run_coordinator.go`
+- `internal/app/upload_coordinator.go`
+- `docs/internal/app.md`
+- `internal/app/run_coordinator_test.go`
+- `internal/app/upload_coordinator_test.go`
+
+Duplicated or near-duplicated behavior:
+
+- Both coordinators define run records, statuses, timestamps, status transitions, context handling, and active pipeline protection.
+- The upload coordinator additionally queues, stages, expires status records, and serializes same-pipeline upload execution.
+
+Why it matters:
+
+- The concepts are similar enough to confuse future contributors.
+- However, the behavior is not identical: one rejects duplicate active runs, while the other queues accepted uploads.
+
+Recommended refactor:
+
+- Do not merge the coordinators now.
+- Review whether `PipelineRunCoordinator` is still needed as an exported app-level helper. If it is intended for future transports, document that role clearly. If not, remove it and its tests in a separate dead-code cleanup.
+- If both remain, extract only tiny shared timestamp/status helpers if a real third coordinator appears.
+
+Suggested tests:
+
+- If retained, keep existing duplicate-run tests.
+- If removed, run `go test ./internal/app ./internal/cli` and verify no current behavior depended on it.
+
+Risk level:
+
+- Low for documentation clarification, medium for removal because it is exported from an internal package and documented for maintainers.
+
+## 5. Boundary and responsibility concerns
+
+The major boundaries are sound:
+
+- CLI parsing stays in `internal/cli`.
+- Config defaults and validation stay in `internal/config`.
+- Backend-specific filesystem, SFTP, and S3 behavior stays in adapters.
+- Manifest semantics are centralized in `pkg/bundle`, with `internal/bundle` adding storage-backed discovery and validation.
+- Destination state comparison stays in `internal/state`.
+- Publish planning/execution stays in `internal/publish`.
+- Transform implementation is behind `internal/transform`.
+
+Concerns worth addressing:
+
+- HTTP upload request-body staging currently crosses the app/ingest boundary. The app transport layer should not own body buffering and archive size enforcement beyond admission and HTTP status projection.
+- Runtime config setup is app-layer behavior, but it is repeated rather than named. A runtime setup helper would clarify the boundary between `internal/config` and command-specific execution.
+- `internal/app/run.go` owns too many destination-loop details. Extracting a destination processing helper would keep orchestration in app while reducing local complexity.
+
+Recommended homes:
+
+- Upload archive policy: `internal/ingest`.
+- HTTP route/auth/status mapping: `internal/app/upload_http.go`.
+- Queueing/status/execution: `internal/app/upload_coordinator.go`.
+- Runtime config plus secret setup: a small helper in `internal/app`, using `internal/config`.
+- Path and state filenames: keep in `internal/storage`.
+
+## 6. Path, key, and naming construction review
+
+Centralized and healthy areas:
+
+- `storage.StateFileName`, `storage.StatePath`, `storage.ManagedBundleTargets`, `storage.Join`, `storage.DisplayPath`, and logical path validation are used in core publication and tests.
+- S3 object-key mapping is contained in `internal/adapters/s3`.
+- SSH and local native path conversion stay inside their adapters.
+- Link URL construction is isolated in `internal/publish/links.go` and URL validation in `internal/link`.
+- Manifest name and schema version are centralized in `pkg/bundle`, with `internal/bundle` aliases.
+
+Areas needing cleanup:
+
+- Archive path cleaning duplicates much of source/storage path policy and should either delegate to a shared primitive or be tightly covered by mirrored tests.
+- Upload run ID construction is isolated, but the shape is partly policy. Keep tests around `<pipeline_id>.<utc_timestamp>.<random_suffix>` before changing coordinator code.
+- Some app tests still construct destination state and source paths locally. `internal/testutil` already covers many cases; additional helper use should be opportunistic, not a sweeping test rewrite.
+
+## 7. Resolution and catalog review
+
+Named concept resolution is mostly consistent:
+
+- Backend names are defined in `internal/config/defaults.go`.
+- Runtime backend construction is app-owned through `backendFactory` and the storage registry.
+- Transform names are defined in `internal/transform`, and app wiring owns concrete registration.
+- Publish/transform policy combinations use `config.ValidatePublishTransformPolicy`.
+- Configured source selection for `validate` and `inspect` is shared in `source_select.go`.
+
+Potential refinements:
+
+- A normalized backend config view would make backend field resolution less repetitive across source and destination config.
+- Transform and backend registries should remain separate; there is no evidence that a generic registry abstraction would help.
+- No separate catalog package is needed for the current feature set.
+
+## 8. Config and command-loading review
+
+Config loading is reliable and strict:
+
+- YAML known-field checking is enabled.
+- Defaults are applied before validation.
+- Validation collects multiple field errors.
+- Secrets are loaded without mutating `os.Environ`.
+- Explicit S3 credential references use the config-owned resolver.
+
+Likely accidental duplication:
+
+- Default config path selection and `config.LoadFile` are repeated in several app entrypoints.
+- Secret loading is repeated in run, source diagnostics, and serve.
+- Secret conflict warning projection is not represented by one runtime setup result.
+
+Intentional differences:
+
+- `serve` loads upload tokens and does not produce CLI JSON output.
+- `validate` and `inspect` support local-path shortcut mode, while `run` and `serve` are config-driven.
+- `manifest create` is local filesystem producer tooling and does not load app config.
+
+Recommended cleanup:
+
+- Centralize runtime config and secret setup in `internal/app`.
+- Keep CLI flag parsing local to command files.
+- Keep `manifest create` outside runtime config loading.
+
+## 9. State, manifest, or progress handling review
+
+Manifest handling is in good shape:
+
+- `pkg/bundle` owns manifest parsing, digest grammar, source path validation, canonical bundle digest, local manifest building, and local bundle writing.
+- `internal/bundle` delegates normalized manifest semantics to `pkg/bundle` and adds storage-backed validation.
+- Destination state embeds the normalized manifest and validates through `internal/bundle`/`pkg/bundle`.
+
+State handling is in good shape:
+
+- `.distributor.json` parsing, validation, JSON projection, and comparison live in `internal/state`.
+- Publish execution writes destination state only after outputs are written.
+- Managed replacement deletes only state-listed outputs plus `.distributor.json`; forced replacement is explicit and bounded.
+
+Progress/status handling:
+
+- `RunReport` is the core run result model and supports JSON partial-result output.
+- HTTP upload status is memory-only and documented as such.
+- `PipelineRunCoordinator` and `UploadCoordinator` overlap conceptually but have different policies. Avoid merging unless product behavior converges.
+
+Gaps:
+
+- HTTP upload staging currently stores the request body in memory before queueing. This is both a quality gap and a mismatch with the intended ingestion boundary.
+- There is no durable upload status, but this is documented as deferred work and should not be addressed in cleanup.
+
+## 10. Refactors to avoid
+
+Avoid these changes in the cleanup pass:
+
+- Do not introduce a generic workflow engine or stage framework. The current explicit workflow is easier to audit.
+- Do not add a CLI framework. The standard-library CLI is sufficient and policy-approved.
+- Do not merge local, SSH, S3, and fake adapters behind a shared implementation layer. Their semantics differ enough that generic helpers would likely hide important behavior.
+- Do not collapse `pkg/bundle` and `internal/bundle`. The public producer API and storage-backed distributor validation have different responsibilities.
+- Do not move destination state comparison into `publish` or app orchestration.
+- Do not redesign JSON output envelopes while doing cleanup.
+- Do not add durable queues, retry workers, HTTP TLS, zstd, or browser UI under the banner of refactoring. These are feature work.
+- Do not rewrite tests wholesale to use a new fixture system. Add helpers only where they reduce immediate duplication around changed code.
+
+## 11. Recommended implementation sequence
+
+1. HTTP upload staging boundary cleanup.
+   - Move supported content-type policy to `internal/ingest`.
+   - Stop buffering accepted uploads in `upload_http.go`.
+   - Queue staged bundle roots rather than request bodies.
+   - Extend HTTP upload tests first.
+
+2. Runtime config setup helper.
+   - Add an app-level helper for default config path, config load, secret load, environment resolver, and secret warnings.
+   - Use it from `run`, configured `validate`/`inspect`, and `serve` where applicable.
+   - Preserve command-specific behavior.
+
+3. Run destination processing extraction.
+   - Add small helpers for destination-scoped failure recording and plan identity normalization.
+   - Extract one-destination processing only if the helper remains readable.
+   - Preserve action ordering and report output.
+
+4. Backend config normalized view.
+   - Add source/destination backend view helpers in `internal/config`.
+   - Use them for defaulting and validation if tests show the shape remains clear.
+   - Keep YAML structs and public config unchanged.
+
+5. Bundle output projection cleanup.
+   - Add app-local helpers for file record and bundle summary projection.
+   - Use `time.RFC3339` consistently.
+   - Preserve command-specific JSON field names.
+
+6. Archive/source path validation test alignment.
+   - Add mirrored path safety tests around ingestion and bundle path validation.
+   - Only centralize code if the helper does not blur archive directory semantics.
+
+7. Coordinator intent cleanup.
+   - Decide whether `PipelineRunCoordinator` is retained for internal future use.
+   - If retained, clarify comments/docs. If removed, do it as a separate dead-code commit.
+
+8. Test helper cleanup.
+   - Expand `internal/testutil` only for repeated setup touched by the previous refactors.
+   - Avoid moving every test fixture.
+
+## 12. Test strategy
+
+Tests to add before refactoring:
+
+- HTTP upload handler test proving queue-full rejection does not consume the body.
+- HTTP upload test proving accepted upload staging completes before `202 Accepted`.
+- Ingestion content-type policy tests exposed through the new helper.
+- Run report test covering destination open failure for multiple selected bundles.
+- CLI JSON tests for `inspect` and `manifest create` timestamp formatting before projection cleanup.
+
+Tests to run with each cleanup stage:
+
+- HTTP upload cleanup: `go test ./internal/ingest ./internal/app ./internal/cli`
+- Config setup cleanup: `go test ./internal/config ./internal/app ./internal/cli`
+- Run processing cleanup: `go test ./internal/app ./internal/publish ./internal/state`
+- Backend config view cleanup: `go test ./internal/config ./internal/app`
+- Output projection cleanup: `go test ./internal/app ./internal/cli`
+- Path validation cleanup: `go test ./pkg/bundle ./internal/bundle ./internal/ingest ./internal/storage`
+- Final cleanup validation: `go test ./...`
+
+Useful read-only checks:
+
+- `rg -n "LoadFile\\(|LoadSecretEnvironment\\(|DefaultConfigPath" internal/app internal/cli`
+- `rg -n "application/x-tar|application/gzip|application/x-gzip" internal docs`
+- `rg -n "2006-01-02T15:04:05Z07:00" internal pkg`
+- `rg -n "manifest.json|\\.distributor.json|StatePath|DisplayPath" internal pkg`
+
+## 13. Appendix: findings not worth acting on
+
+Adapter `ReadFile` and `WriteFile` wrappers:
+
+- Local, SSH, S3, and fake backends each implement byte helpers in terms of stream helpers. This is small duplication but appropriate because each adapter owns error translation and metadata semantics.
+
+Adapter traversal implementation:
+
+- Local filesystem walking, SFTP walking, and S3 pagination look similar at the interface level but are semantically different. Keep traversal mechanics in adapters and shared callback behavior in `storage.WalkEmitter`.
+
+State and manifest raw JSON parsing:
+
+- `pkg/bundle` and `internal/state` both parse raw JSON with pointer fields to detect missing required fields. The schemas and error contexts differ, so a generic required-field parser would not be worth the complexity.
+
+CLI help text:
+
+- Help text repeats command names and flags. This is acceptable in a small hand-written CLI and keeps command files readable.
+
+Test fixture strings:
+
+- Some tests inline YAML snippets or expected output strings despite `internal/testutil`. Inline data is often clearer for edge cases. Only centralize fixture setup when tests are already being changed for a behavior-preserving refactor.
+
+HTTP JSON response helpers:
+
+- HTTP API responses use simple JSON objects rather than the CLI JSON envelope. This is intentional because HTTP status codes and route-specific responses are not the same public interface as CLI command output.
+
+Public and internal bundle validation:
+
+- `pkg/bundle.ValidateBundle` is local-filesystem producer validation; `internal/bundle.Validate` is storage-backed distributor validation. Keep both, with shared manifest semantics delegated through `pkg/bundle`.

docs/roadmap/cleanup.md

@@ -0,0 +1,400 @@
+# Code Quality Cleanup Roadmap
+
+## Current Baseline
+
+The codebase has completed the local, SSH/SFTP, S3, public bundle package,
+manifest creation, JSON output, path mapping, link generation, and HTTP upload
+work documented in the current user and internal docs.
+
+The audit in `docs/roadmap/audit.md` found no major architectural risk. The
+remaining cleanup work should be narrow, behavior-preserving, and focused on
+reducing drift in upload staging, runtime config setup, run reporting, backend
+config handling, output projection, path validation tests, and internal
+coordination code.
+
+One intentional behavior change is part of this cleanup roadmap: malformed
+authenticated upload archives should be rejected before `202 Accepted`, rather
+than accepted and later marked failed. Valid staged uploads should still run
+asynchronously after admission.
+
+## Cleanup Principles
+
+- Preserve public CLI behavior, config schema, manifest schema, destination
+  state schema, backend behavior, and JSON envelopes unless a stage explicitly
+  says otherwise.
+- Keep config parsing and validation in `internal/config`.
+- Keep CLI parsing in `internal/cli`.
+- Keep upload archive policy in `internal/ingest`; keep HTTP routing,
+  authentication, and status projection in `internal/app`.
+- Keep backend-specific filesystem, SSH/SFTP, and S3 behavior in adapter
+  packages.
+- Prefer small package-local helpers over broad abstractions.
+- Add or strengthen tests before refactoring behavior that affects public
+  output, upload admission, path safety, or run reporting.
+
+## Active Cleanup Stages
+
+Implement these stages in order. Each stage should be small enough for one
+implementation prompt and should leave the repository passing the listed focused
+tests before moving to the next stage.
+
+## Stage 1: HTTP Upload Staging Boundary
+
+Goal:
+
+Move archive validation and upload body staging fully behind `internal/ingest`,
+stop app-layer full-body buffering, and reject malformed archives before
+returning `202 Accepted`.
+
+Implementation scope:
+
+- Add an ingestion-owned content-type helper, such as
+  `ValidateContentType(contentType string) error`, and remove duplicated
+  content-type policy from the HTTP handler.
+- Replace the current handler-side `readUploadBody` buffering with streaming
+  staging through `internal/ingest`.
+- Introduce a two-step upload coordinator admission model:
+  - reserve a run id and queue slot before consuming the request body;
+  - stage and validate the archive using that reserved run id;
+  - enqueue only a successfully staged local bundle root for async execution.
+- Keep queue-full rejection before reading the body.
+- Preserve `401` for missing or invalid bearer tokens, `415` for unsupported
+  content type, `413` for oversized uploads, and `503` for a full queue.
+- Return a pre-acceptance `400` for malformed tar/gzip content or invalid
+  staged bundles.
+- Preserve async queued/running/succeeded/failed status after a valid staged
+  bundle is accepted.
+- Do not add durable queues, idempotency keys, zstd, or new routes.
+
+Current-behavior documentation updates:
+
+- Update `docs/cli.md`, `docs/config.md`, `docs/operations.md`,
+  `docs/troubleshooting.md`, `docs/internal/app.md`, and
+  `docs/internal/ingest.md` only as needed to describe the new
+  pre-acceptance failure boundary.
+
+Tests:
+
+- `go test ./internal/ingest ./internal/app ./internal/cli`
+- Queue-full upload rejection does not read the request body.
+- Unsupported content type is rejected through ingestion-owned policy.
+- Oversized uploads return `413` and do not retain a staged run.
+- Malformed tar/gzip content returns `400` before a run id is issued.
+- Valid tar and tar.gz uploads return `202` after staging and still transition
+  through async status.
+- HTTP responses and status records do not leak bearer tokens or secret values.
+
+Completion criteria:
+
+- `internal/app` no longer buffers the full upload body before staging.
+- A valid accepted upload has a committed staged bundle root before the `202`
+  response is sent.
+- Invalid archive content cannot create an accepted run id.
+
+## Stage 2: Runtime Config And Secret Setup Helper
+
+Goal:
+
+Centralize runtime config path resolution, config loading, secret loading,
+environment resolver creation, and secret-conflict warning projection in one
+app-layer helper.
+
+Implementation scope:
+
+- Add a small `internal/app` runtime setup helper that:
+  - defaults an empty config path to `config.DefaultConfigPath`;
+  - calls `config.LoadFile`;
+  - calls `config.LoadSecretEnvironment`;
+  - exposes the loaded config, config path, `config.Environment`, and
+    `[]OutputWarning` for secret conflicts.
+- Use the helper from `Run`, `RunPipeline`, `RunPipelineWithLocalSource`,
+  configured `Validate`/`Inspect`, and `Serve` where applicable.
+- Keep `manifest create` outside runtime config loading.
+- Keep YAML structs, defaults, validation, and secret-directory parsing in
+  `internal/config`.
+- Preserve app test injection points for backend factories and upload handler
+  tests.
+
+Current-behavior documentation updates:
+
+- Update `docs/internal/app.md` if helper boundaries or flow descriptions
+  change. User-facing docs should not change unless observable behavior changes.
+
+Tests:
+
+- `go test ./internal/config ./internal/app ./internal/cli`
+- Default config path behavior remains unchanged.
+- Secret conflict warnings still appear in text and JSON output for `run`,
+  configured `validate`, and configured `inspect`.
+- `serve` still fails startup safely for missing, empty, or duplicate upload
+  tokens without leaking values.
+- Explicit S3 credential references still resolve through the config-owned
+  environment resolver.
+
+Completion criteria:
+
+- Runtime commands no longer repeat config path defaulting and secret loading.
+- Config policy remains owned by `internal/config`.
+
+## Stage 3: Run Destination Processing Extraction
+
+Goal:
+
+Reduce complexity in the main run loop while preserving run report behavior,
+warning ordering, action ordering, failure aggregation, and text/JSON output.
+
+Implementation scope:
+
+- Extract narrow helpers from `internal/app/run.go` for destination-scoped
+  processing.
+- Centralize destination-scoped failure recording so one helper updates
+  `runFailures`, `runSummary`, `RunReport.Actions`, and pipeline events.
+- Centralize normalization of partial `publish.Plan` identity fields before
+  converting plans to run action records.
+- Keep app orchestration explicit; do not introduce a generic workflow engine,
+  stage framework, or broad runner abstraction.
+- Preserve independent destination fan-out and partial-result behavior.
+
+Current-behavior documentation updates:
+
+- Update `docs/internal/app.md` only if helper names or package layout
+  descriptions materially change.
+
+Tests:
+
+- `go test ./internal/app ./internal/publish ./internal/state`
+- Destination open failures for multiple selected bundles keep action records,
+  output errors, summary counters, and pipeline events aligned.
+- JSON partial-result output remains unchanged when destination planning or
+  execution fails after a report exists.
+- Fixed-path dry-run warnings appear in the same order as before.
+- Existing run text output assertions continue to pass.
+
+Completion criteria:
+
+- `run.go` delegates destination-scoped record/failure bookkeeping to helpers.
+- No public output shape or ordering changes.
+
+## Stage 4: Backend Config Normalized View
+
+Goal:
+
+Reduce source/destination backend config drift while preserving the current YAML
+schema and public config behavior.
+
+Implementation scope:
+
+- Add package-local normalized backend view helpers in `internal/config` for
+  source and destination backend fields.
+- Use the normalized view to reduce duplication in backend defaulting and
+  validation where it remains clearer than the current paired code.
+- Keep `config.Backend` and `config.Destination` YAML structs and tags
+  unchanged.
+- Preserve destination-only policy fields on `Destination`.
+- Preserve `http_upload` as source-only and invalid for destinations.
+- Update app backend opening only if the normalized view provides clearer
+  handoff without leaking config internals.
+
+Current-behavior documentation updates:
+
+- None expected unless internal docs mention the old paired implementation
+  shape in a way that becomes misleading.
+
+Tests:
+
+- `go test ./internal/config ./internal/app`
+- Equivalent local, SSH, and S3 source/destination validation remains
+  consistent.
+- Defaults for SSH port/host key policy, S3 region/prefix/force-path-style, and
+  HTTP upload staging fields remain unchanged.
+- `http_upload` remains valid only for sources.
+
+Completion criteria:
+
+- Adding a future backend field has one obvious defaulting/validation path.
+- Public config files and examples continue to load unchanged.
+
+## Stage 5: Command Output Projection Cleanup
+
+Goal:
+
+Reduce drift in bundle and file metadata projection for app command JSON
+results.
+
+Implementation scope:
+
+- Add small app-local projection helpers for bundle summaries and manifest file
+  records used by `validate`, `inspect`, and `manifest create`.
+- Use `time.RFC3339` consistently instead of equivalent literal layouts.
+- Preserve existing JSON envelope fields, command names, command-specific result
+  field names, text output, and fatal error behavior.
+- Do not redesign CLI JSON output or HTTP JSON responses.
+
+Current-behavior documentation updates:
+
+- None expected unless tests reveal current docs are stale.
+
+Tests:
+
+- `go test ./internal/app ./internal/cli`
+- JSON output for `validate`, `inspect`, and `manifest create` remains
+  structurally stable.
+- RFC3339 timestamps remain unchanged, including offset-preserving source
+  timestamps where current behavior preserves them.
+- Text output remains unchanged.
+
+Completion criteria:
+
+- Bundle/file projection logic is shared where semantics match.
+- Command-specific result structs remain easy to read.
+
+## Stage 6: Archive And Source Path Validation Alignment
+
+Goal:
+
+Protect path safety by aligning archive path tests with source and storage path
+policy, without blurring archive-specific rules.
+
+Implementation scope:
+
+- Add mirrored path-safety table tests around `internal/ingest`, `pkg/bundle`,
+  `internal/bundle`, and `internal/storage` where useful.
+- Keep archive-specific directory handling, root-level manifest rules, duplicate
+  file rejection, symlink rejection, hardlink rejection, and special-entry
+  rejection in `internal/ingest`.
+- Centralize code only if the helper can preserve clear archive semantics and
+  current error behavior.
+- Do not add new public `pkg/bundle` APIs unless the existing public API cannot
+  safely support the needed shared behavior.
+
+Current-behavior documentation updates:
+
+- None expected unless implementation changes error boundaries or internal
+  package descriptions.
+
+Tests:
+
+- `go test ./pkg/bundle ./internal/bundle ./internal/ingest ./internal/storage`
+- Absolute paths, traversal, backslashes, empty paths, dot segments, nested
+  manifests, `.distributor.json` handling, symlinks, hardlinks, devices, and
+  sockets remain covered.
+- Archive directories remain accepted where safe.
+
+Completion criteria:
+
+- Path safety policy has regression coverage across archive staging, source
+  bundle validation, and storage logical path validation.
+- Any code sharing is smaller and clearer than the duplicated logic it replaces.
+
+## Stage 7: Pipeline Run Coordinator Removal
+
+Goal:
+
+Remove the currently unused internal `PipelineRunCoordinator` to avoid
+maintaining two similar coordination concepts.
+
+Implementation scope:
+
+- Delete `PipelineRunCoordinator`, `PipelineRunRecord`,
+  `DuplicatePipelineRunError`, related helpers, and their tests.
+- Remove or rewrite `docs/internal/app.md` sections that describe the removed
+  coordinator.
+- Keep `UploadCoordinator`; do not merge upload queueing with the removed
+  duplicate-run coordinator.
+- Before deletion, confirm with `rg` that production code does not reference
+  `NewPipelineRunCoordinator`, `PipelineRunCoordinator`, or
+  `DuplicatePipelineRunError`.
+
+Current-behavior documentation updates:
+
+- Update `docs/internal/app.md` because it currently documents the coordinator
+  as an internal implemented component.
+
+Tests:
+
+- `go test ./internal/app ./internal/cli`
+- `rg -n "PipelineRunCoordinator|NewPipelineRunCoordinator|DuplicatePipelineRunError" internal docs`
+  should show no stale references after removal.
+
+Completion criteria:
+
+- No production, test, or internal documentation references remain for the
+  removed coordinator.
+- Upload coordination behavior is unchanged.
+
+## Stage 8: Narrow CLI And Test Helper Cleanup
+
+Goal:
+
+Apply only low-risk CLI setup and test fixture cleanup that remains useful after
+the earlier stages.
+
+Implementation scope:
+
+- Add tiny CLI helpers for repeated `flag.FlagSet` setup or output-format
+  parsing only where command behavior remains obvious.
+- Keep the standard-library CLI; do not introduce a CLI framework.
+- Keep `manifest create` interspersed positional parsing local unless another
+  command now needs the same parsing behavior.
+- Expand `internal/testutil` only for repeated setup touched by earlier stages.
+- Do not rewrite tests wholesale just to use shared helpers.
+
+Current-behavior documentation updates:
+
+- None expected unless CLI help or syntax changes. This stage should avoid such
+  changes.
+
+Tests:
+
+- `go test ./internal/cli ./internal/app`
+- CLI usage-error tests remain stable.
+- `manifest create <path> --id x`, `manifest create --id x <path>`, missing
+  flag values, invalid `--format`, and help output remain covered.
+
+Completion criteria:
+
+- Remaining CLI/test cleanup is small, readable, and behavior-preserving.
+- No public CLI syntax or output changes.
+
+## Refactors To Avoid
+
+- Do not introduce a generic workflow engine or stage framework.
+- Do not add a CLI framework.
+- Do not merge local, SSH, S3, and fake backend adapter implementations.
+- Do not collapse `pkg/bundle` and `internal/bundle`.
+- Do not move destination state comparison into `internal/publish` or
+  `internal/app`.
+- Do not redesign CLI JSON envelopes.
+- Do not change HTTP JSON response shapes except where Stage 1 requires
+  pre-acceptance error behavior.
+- Do not add durable upload queues, retry workers, zstd support, in-app TLS,
+  idempotency keys, browser UI, or other feature work.
+- Do not rewrite tests wholesale to use new fixture helpers.
+
+## Validation
+
+After each implementation stage, run the stage-specific tests listed above.
+
+After all cleanup stages:
+
+```sh
+go test ./...
+```
+
+Recommended consistency checks:
+
+```sh
+rg -n "LoadFile\\(|LoadSecretEnvironment\\(|DefaultConfigPath" internal/app internal/cli
+rg -n "application/x-tar|application/gzip|application/x-gzip" internal docs
+rg -n "2006-01-02T15:04:05Z07:00" internal pkg
+rg -n "PipelineRunCoordinator|NewPipelineRunCoordinator|DuplicatePipelineRunError" internal docs
+```
+
+The cleanup is complete when:
+
+- all staged tests and `go test ./...` pass;
+- current-behavior docs describe the implemented Stage 1 upload failure
+  boundary;
+- `docs/roadmap/audit.md` findings have either been addressed or consciously
+  left in place as noted in this cleanup roadmap;
+- no completed cleanup behavior is documented only as future work.

docs/roadmap/http.md

@@ -0,0 +1,60 @@
+# Roadmap: HTTP Upload Extensions
+
+## Purpose
+
+The HTTP upload API is implemented. Current behavior is documented in:
+
+- [CLI](../cli.md)
+- [Configuration](../config.md)
+- [Operations](../operations.md)
+- [Troubleshooting](../troubleshooting.md)
+- [Application internals](../internal/app.md)
+- [Ingestion internals](../internal/ingest.md)
+
+This roadmap records HTTP upload extensions that are intentionally not part of
+the current implementation.
+
+## Deferred Extensions
+
+### Authentication
+
+- URL-token authentication for constrained clients.
+- Additional token lifecycle tooling.
+- Mutual TLS or other in-app identity mechanisms.
+
+### Archive Formats
+
+- Zstandard-compressed tar archives.
+- Additional content negotiation rules for future archive formats.
+
+### Status And Queue Durability
+
+- Durable status persistence across process restarts.
+- Database-backed queueing.
+- Recovery semantics for queued or running uploads after a restart.
+
+### Producer Coordination
+
+- Producer-supplied idempotency keys.
+- Run retry endpoints.
+- Run cancellation endpoints.
+- Run listing endpoints.
+
+### Deployment Surface
+
+- In-app TLS.
+- Public exposure defaults.
+- Browser UI.
+
+## Boundaries
+
+Current HTTP upload behavior remains intentionally small:
+
+- `http_upload` is source-only and is not a durable storage backend.
+- Upload status is memory-only.
+- Producers submit complete tar or gzip-compressed tar source bundles.
+- Producers authenticate with `Authorization: Bearer <token>`.
+- Public access policy, TLS termination, and rate limiting belong outside
+  `distributor` unless a future roadmap explicitly changes that boundary.
+
+Do not document deferred extensions as available outside `docs/roadmap/`.

docs/roadmap/implementation.md

@@ -1,102 +1,34 @@
-# Roadmap
+# HTTP Upload Deferred Work
 
-This directory contains only future, deferred, or aspirational work for
+## Purpose
-`distributor`. Implemented behavior is documented in the current user,
-operator, internal, policy, integration, and example documentation:
 
-- `README.md`
+HTTP upload behavior is implemented and documented in the current-behavior
-- `docs/cli.md`
+manuals:
-- `docs/config.md`
-- `docs/operations.md`
-- `docs/troubleshooting.md`
-- `docs/internal/`
-- `docs/integrations/markdown.md`
-- `docs/policy/`
-- `examples/`
 
-`distributor` currently supports local, SSH/SFTP, and S3-compatible source and
+- [CLI](../cli.md)
-destination backends; producer bundle creation through `pkg/bundle` and
+- [Configuration](../config.md)
-`distributor manifest create`; configured source validation and inspection;
+- [Operations](../operations.md)
-Markdown sidecar and `index.html` publication; archive and fixed destination
+- [Troubleshooting](../troubleshooting.md)
-path mapping; destination link metadata; shared text/JSON CLI output; and
+- [Application internals](../internal/app.md)
-managed destination replacement behavior.
+- [Configuration internals](../internal/config.md)
+- [Ingestion internals](../internal/ingest.md)
 
-## Future Work
+This file tracks only HTTP upload work that is not implemented.
 
-These items are not implemented. They should not be documented as current
+## Deferred Work
-behavior outside `docs/roadmap/` unless a future implementation adds them.
 
-### CLI And Status Output
+- URL-token authentication.
+- Zstandard-compressed archive support.
+- Durable status persistence across process restarts.
+- Database-backed queueing.
+- Producer-supplied idempotency keys.
+- Run listing, cancellation, and retry endpoints.
+- In-app TLS.
+- Public network exposure defaults.
+- Browser UI.
 
-- Add a root-global output flag only if the command parser is later refactored
+## Documentation Rule
-  around shared root options.
-- Add output formats beyond `text` and `json` only if a concrete consumer
-  requires them.
-- Add a versioned JSON schema reference after the first JSON-capable release.
-- Add destination-state inspection behind an explicit flag such as
-  `--with-destinations` if operators need fan-out status diagnostics from
-  `inspect`.
-- Add additional status or inspection presentation for destination primary
-  links beyond the current `run --format json` result model.
 
-### Producer Workflows
+Deferred behavior belongs under `docs/roadmap/` until implemented. Current
-
+behavior docs must describe only the active HTTP upload API, configuration,
-- Add a no-write manifest creation mode, such as writing manifest JSON to
+operation, troubleshooting, and internal package contracts.
-  stdout, if producer pipelines need to capture manifests directly.
-- Add broader producer workflow helpers, such as richer ignore rules or
-  template scaffolding, if real producer use cases require them.
-- Add remote or storage-backed producer writers only if producer applications
-  need to assemble bundles outside the local filesystem.
-
-### Publication And Transform Behavior
-
-- Add a separate collection or site-index transform if distributor needs
-  multi-page aggregation.
-- Add richer transform metadata only if future state consumers need more than
-  the transform name and output path.
-- Add custom HTML index output names only if fixed `index.html` is too limiting
-  for real deployments.
-- Add richer fixed-destination source selection policies if deployments need
-  something other than newest-by-`created`.
-- Add stricter handling for equal latest timestamps if timestamp ties become
-  common in producer workflows.
-- Add higher-level status or approval workflows for fixed-root replacements if
-  dry-run output is not enough operational protection.
-- Add richer link policies only if `auto`, `html`, and `source` prove
-  insufficient.
-
-### State And Compatibility
-
-- Define a post-release destination state schema bump policy before introducing
-  materially incompatible state changes.
-- Add warning-only digest mismatch handling only if an operator workflow needs
-  publication to continue after validation failures.
-- Add compatibility parsing for legacy SSH URI config only if migration support
-  is required.
-
-### Backends, Security, And Deployment
-
-- Add authentication mechanisms beyond the implemented SSH agent/key and S3
-  credential paths only when a concrete backend workflow requires them.
-- Add broad recursive destination deletion outside managed bundle paths only if
-  a future design can preserve the current safety boundary.
-- Add concurrent fan-out publishing only if runtime profiling shows it is
-  needed.
-- Add streaming, resumable, or multipart S3 uploads only if object sizes make
-  the current write path insufficient.
-- Add cloud-provider-specific IAM integration docs only when the repository
-  includes tested provider-specific behavior.
-- Add repository-managed packaging, release, and deployment automation when the
-  release process is ready to be standardized.
-
-## Roadmap Maintenance
-
-When adding future roadmap work:
-
-- describe user-visible behavior and safety boundaries;
-- define which current docs must change after implementation;
-- keep examples secret-free and runnable or clearly environment-gated;
-- keep workflow labels out of production code, tests, config fields, and
-  user-facing documentation;
-- run focused tests for the changed behavior and `go test ./...` for
-  cross-package changes.

docs/troubleshooting.md

@@ -26,7 +26,10 @@ Safe fix: compare the file to the reference in [configuration](config.md) and re
 
 ## `validate config ... backend ... is unsupported`
 
-Likely cause: a source or destination uses a backend name other than `local`, `ssh`, or `s3`.
+Likely cause: a source or destination uses an unsupported backend name, or a
+command is trying to execute a backend that is valid only for another workflow.
+`run`, `validate`, and `inspect` execute `local`, `ssh`, and `s3` sources.
+`serve` executes `http_upload` sources.
 
 Diagnostic:
 
@@ -34,7 +37,127 @@ Diagnostic:
 rg -n "backend:" <config-path>
 ```
 
-Safe fix: use `backend: local`, `backend: ssh`, or `backend: s3` for executable workflows.
+Safe fix: use `backend: local`, `backend: ssh`, or `backend: s3` for normal
+source and destination workflows. Use `backend: http_upload` only for sources
+handled by `distributor serve`.
+
+## `bind HTTP server ... address already in use`
+
+Likely cause: another process is already listening on `server.http.bind`.
+
+Diagnostic:
+
+```sh
+ss -ltnp | rg '<port>'
+```
+
+Safe fix: stop the conflicting process or configure a different
+`server.http.bind` value. The default bind address is `127.0.0.1:8080`.
+
+## `upload token environment variable ... is not set`
+
+Likely cause: a configured `http_upload` source references `token_env`, but the
+variable is absent from both the real process environment and
+`secrets.directory`.
+
+Diagnostic:
+
+```sh
+env | cut -d= -f1 | rg '^<token-variable>$'
+ls -l <secrets-directory>/<token-variable>
+```
+
+Safe fix: set the real environment variable or create a readable
+secrets-directory file with the same name. Do not place literal token values in
+YAML.
+
+## `upload token environment variables ... resolve to the same value`
+
+Likely cause: two configured `http_upload` pipelines resolve to the same bearer
+token value.
+
+Diagnostic:
+
+```sh
+rg -n 'token_env:' <config-path>
+```
+
+Safe fix: assign a distinct non-empty token value to each `http_upload`
+pipeline. Distributor does not print the duplicate token value.
+
+## `POST /upload` returns `401`
+
+Likely cause: the request is missing `Authorization: Bearer <token>` or the
+token does not match any configured `http_upload` pipeline.
+
+Diagnostic:
+
+```sh
+curl -i -X POST http://127.0.0.1:8080/upload \
+  -H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
+  -H "Content-Type: application/x-tar" \
+  --data-binary @bundle.tar
+```
+
+Safe fix: use the token value resolved by the configured `token_env`. Do not
+include token values in logs or tickets.
+
+## `POST /upload` returns `413`
+
+Likely cause: the request body exceeds the selected pipeline's
+`source.max_upload_size` or the default `server.http.max_upload_size`.
+
+Diagnostic:
+
+```sh
+ls -lh bundle.tar bundle.tar.gz
+rg -n 'max_upload_size:' <config-path>
+```
+
+Safe fix: upload a smaller archive, remove unnecessary files from the source
+bundle, or raise the configured upload size limit.
+
+## `POST /upload` returns `415`
+
+Likely cause: the upload uses an unsupported content type. The server accepts
+uncompressed tar and gzip-compressed tar archives only.
+
+Diagnostic:
+
+```sh
+file bundle.tar.gz
+```
+
+Safe fix: send `Content-Type: application/x-tar`, `application/gzip`, or
+`application/x-gzip`, matching the archive format.
+
+## `POST /upload` returns `503`
+
+Likely cause: the in-memory upload queue is full.
+
+Diagnostic:
+
+```sh
+rg -n 'queue_size|max_concurrency' <config-path>
+```
+
+Safe fix: retry after active uploads finish, or increase `server.http.queue_size`
+for the deployment.
+
+## `GET /runs/<run_id>` returns `404`
+
+Likely cause: the run id is wrong, the process restarted, or the completed
+status record expired after `server.http.retention`.
+
+Diagnostic:
+
+```sh
+curl -i http://127.0.0.1:8080/runs/<run-id>
+rg -n 'retention:' <config-path>
+```
+
+Safe fix: use the exact `run_id` returned by `POST /upload`. If status retention
+is too short for operators, increase `server.http.retention`.
 
 ## `--format: format must be text or json`
 

examples/http-upload-local.yml

@@ -0,0 +1,24 @@
+# Local HTTP upload example.
+# Set DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN in the process environment or provide a
+# secrets-directory file with that name before running `distributor serve`.
+server:
+  http:
+    bind: 127.0.0.1:8080
+    staging_root: workspace/http-upload/staging
+    max_upload_size: 20MB
+    queue_size: 16
+    max_concurrency: 1
+    retention: 24h
+pipelines:
+  - id: example-http-upload
+    source:
+      backend: http_upload
+      token_env: DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN
+    destinations:
+      - id: local-archive
+        backend: local
+        path: workspace/published/http-upload
+        publish:
+          source: true
+          html: false
+

internal/app/run.go

@@ -21,6 +21,23 @@ type RunOptions struct {
 	Notifier     notify.Notifier
 }
 
+type RunPipelineOptions struct {
+	ConfigPath string
+	PipelineID string
+	DryRun     bool
+	Force      bool
+	Notifier   notify.Notifier
+}
+
+type RunPipelineWithLocalSourceOptions struct {
+	ConfigPath string
+	PipelineID string
+	SourceRoot string
+	DryRun     bool
+	Force      bool
+	Notifier   notify.Notifier
+}
+
 func Run(ctx context.Context, options RunOptions) error {
 	if err := ValidateOutputFormat(options.OutputFormat); err != nil {
 		return err
@@ -40,90 +57,155 @@ func Run(ctx context.Context, options RunOptions) error {
 	return runConfig(ctx, cfg, options)
 }
 
+func RunPipeline(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
+	if err := ctx.Err(); err != nil {
+		return RunReport{}, err
+	}
+
+	configPath := options.ConfigPath
+	if configPath == "" {
+		configPath = config.DefaultConfigPath
+	}
+	cfg, err := config.LoadFile(configPath)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return runPipelineConfig(ctx, cfg, options)
+}
+
+func RunPipelineWithLocalSource(ctx context.Context, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+	if err := ctx.Err(); err != nil {
+		return RunReport{}, err
+	}
+	if options.SourceRoot == "" {
+		return RunReport{}, fmt.Errorf("source root is required")
+	}
+
+	configPath := options.ConfigPath
+	if configPath == "" {
+		configPath = config.DefaultConfigPath
+	}
+	cfg, err := config.LoadFile(configPath)
+	if err != nil {
+		return RunReport{}, err
+	}
+	return runPipelineConfigWithLocalSource(ctx, cfg, options)
+}
+
 func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error {
 	return runConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
 }
 
 type backendFactoryProvider func(config.Environment) *backendFactory
 
+func runPipelineConfig(ctx context.Context, cfg config.Config, options RunPipelineOptions) (RunReport, error) {
+	return runPipelineConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
+}
+
+func runPipelineConfigWithLocalSource(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+	return runPipelineConfigWithLocalSourceAndBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
+}
+
+func runPipelineConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineOptions, provider backendFactoryProvider) (RunReport, error) {
+	pipeline, ok := findPipeline(cfg, options.PipelineID)
+	if !ok {
+		return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
+	}
+	return buildRunReportWithBackendFactory(ctx, config.Config{
+		Server:    cfg.Server,
+		Secrets:   cfg.Secrets,
+		Pipelines: []config.Pipeline{pipeline},
+	}, RunOptions{
+		DryRun:   options.DryRun,
+		Force:    options.Force,
+		Notifier: options.Notifier,
+	}, provider)
+}
+
+func runPipelineConfigWithLocalSourceAndBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions, provider backendFactoryProvider) (RunReport, error) {
+	pipeline, ok := findPipeline(cfg, options.PipelineID)
+	if !ok {
+		return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
+	}
+	return buildRunReport(ctx, config.Config{
+		Server:    cfg.Server,
+		Secrets:   cfg.Secrets,
+		Pipelines: []config.Pipeline{pipeline},
+	}, RunOptions{
+		DryRun:   options.DryRun,
+		Force:    options.Force,
+		Notifier: options.Notifier,
+	}, provider, &localSourceRoot{
+		pipelineID: options.PipelineID,
+		root:       options.SourceRoot,
+	})
+}
+
 func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) error {
+	report, err := buildRunReportWithBackendFactory(ctx, cfg, options, provider)
+	if err != nil && !IsPartialResultError(err) {
+		return err
+	}
+	if outputErr := WriteRunReport(options.Stdout, options.OutputFormat, report); outputErr != nil {
+		return outputErr
+	}
+	return err
+}
+
+func buildRunReportWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) (RunReport, error) {
+	return buildRunReport(ctx, cfg, options, provider, nil)
+}
+
+type localSourceRoot struct {
+	pipelineID string
+	root       string
+}
+
+func buildRunReport(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider, sourceRoot *localSourceRoot) (RunReport, error) {
 	notifier := options.Notifier
 	if notifier == nil {
 		notifier = notify.Noop{}
 	}
-	jsonOutput := IsJSONOutput(options.OutputFormat)
 	summary := runSummary{dryRun: options.DryRun}
-	result := runResult{
+	report := RunReport{
 		DryRun:    options.DryRun,
-		Pipelines: []runPipelineResult{},
+		Pipelines: []RunPipelineSummary{},
-		Actions:   []runActionResult{},
+		Actions:   []RunActionRecord{},
 	}
-	var warnings []OutputWarning
 	var failures runFailures
 	secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
 	if err != nil {
-		return err
+		return report, err
 	}
 	secretWarnings := secretConflictWarnings(secretLoad.Conflicts)
-	if jsonOutput {
+	report.PreambleWarnings = append(report.PreambleWarnings, secretWarnings...)
-		warnings = append(warnings, secretWarnings...)
+	report.addWarnings(secretWarnings)
-	} else if options.Stdout != nil {
-		if err := writeWarnings(options.Stdout, secretWarnings); err != nil {
-			return err
-		}
-	}
 	backends := provider(secretLoad.Environment)
 	backends.readOnlyKnownHosts = options.DryRun
 	transforms := newTransformRegistry()
-	if options.Stdout != nil && !jsonOutput {
-		if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
-			return err
-		}
-	}
 	for _, pipeline := range cfg.Pipelines {
 		pipelineWarnings := sshWarnings(pipeline)
-		if jsonOutput {
+		report.addWarnings(pipelineWarnings)
-			warnings = append(warnings, pipelineWarnings...)
+		sourceBackend, bundles, sourceBackendName, err := openPipelineSource(ctx, backends, pipeline, sourceRoot)
-		} else if options.Stdout != nil {
-			if err := writeWarnings(options.Stdout, pipelineWarnings); err != nil {
-				return err
-			}
-		}
-		sourceBackend, err := backends.openSource(ctx, pipeline.Source)
 		if err != nil {
-			return fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
+			return report, err
 		}
-		bundles, err := bundle.Discover(ctx, sourceBackend, "")
+		report.Pipelines = append(report.Pipelines, RunPipelineSummary{
-		if err != nil {
-			closeBackend(sourceBackend)
-			return fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
-		}
-		result.Pipelines = append(result.Pipelines, runPipelineResult{
 			ID:            pipeline.ID,
-			SourceBackend: pipeline.Source.Backend,
+			SourceBackend: sourceBackendName,
 			BundleCount:   len(bundles),
 			Destinations:  destinationIDs(pipeline.Destinations),
+			Warnings:      pipelineWarnings,
 		})
-		if options.Stdout != nil && !jsonOutput {
+		pipelineIndex := len(report.Pipelines) - 1
-			if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
-				closeBackend(sourceBackend)
-				return err
-			}
-		}
 		for _, destination := range pipeline.Destinations {
 			selections := selectDestinationBundles(destination, bundles)
 			if isFixedPathDestination(destination) {
 				summary.recordFixedPath()
 				if options.DryRun {
 					warning := fixedPathSelectionWarning(pipeline.ID, destination.ID, selections, len(bundles))
-					if jsonOutput {
+					report.addWarning(warning)
-						warnings = append(warnings, warning)
+					report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, warningEvent(warning))
-					} else if options.Stdout != nil {
-						if err := writeWarnings(options.Stdout, []OutputWarning{warning}); err != nil {
-							closeBackend(sourceBackend)
-							return err
-						}
-					}
 				}
 			}
 			if len(selections) == 0 {
@@ -134,11 +216,8 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
 				for _, selection := range selections {
 					failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(selection.SourceBundle.RootRelativePath), err)
 					summary.recordFailure()
-					if jsonOutput {
+					report.Actions = append(report.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, selection.SourceBundle.RootRelativePath, err))
-						result.Actions = append(result.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, selection.SourceBundle.RootRelativePath, err))
+					report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, actionEvent(len(report.Actions)-1))
-					} else if options.Stdout != nil {
-						writeErrorLine(options.Stdout, selection.SourceBundle.RootRelativePath, destination.ID, destination.Backend, err)
-					}
 				}
 				continue
 			}
@@ -189,22 +268,12 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
 					plan.PathMapping = config.PathMappingFixed
 					if options.DryRun && isDestructiveFixedPathAction(plan.Action) {
 						warning := fixedPathReplacementWarning(plan)
-						if jsonOutput {
+						report.addWarning(warning)
-							warnings = append(warnings, warning)
+						report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, warningEvent(warning))
-						} else if options.Stdout != nil {
-							if err := writeWarnings(options.Stdout, []OutputWarning{warning}); err != nil {
-								deferCloseDestination()
-								closeBackend(sourceBackend)
-								return err
-							}
-						}
 					}
 				}
-				if jsonOutput {
+				report.Actions = append(report.Actions, runActionFromPlan(destination.Backend, plan, err))
-					result.Actions = append(result.Actions, runActionFromPlan(destination.Backend, plan, err))
+				report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, actionEvent(len(report.Actions)-1))
-				} else if options.Stdout != nil {
-					writePlanLine(options.Stdout, destination.Backend, plan, err)
-				}
 				if err != nil {
 					failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
 					summary.recordFailure()
@@ -230,20 +299,38 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
 		}
 		closeBackend(sourceBackend)
 	}
-	result.Summary = summary.Result()
+	report.Summary = summary.Result()
-	if jsonOutput {
+	report.OutputErrors = failures.outputErrors()
-		if err := WriteJSONEnvelope(options.Stdout, "run", len(failures.items) == 0, warnings, result, failures.outputErrors()); err != nil {
-			return err
-		}
-	} else if options.Stdout != nil {
-		if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil {
-			return err
-		}
-	}
 	if len(failures.items) > 0 {
-		return failures
+		return report, failures
 	}
-	return nil
+	return report, nil
+}
+
+func openPipelineSource(ctx context.Context, backends *backendFactory, pipeline config.Pipeline, sourceRoot *localSourceRoot) (storage.Backend, []bundle.Bundle, string, error) {
+	if sourceRoot != nil && sourceRoot.pipelineID == pipeline.ID {
+		sourceBackend, err := backends.openLocalPath(ctx, sourceRoot.root)
+		if err != nil {
+			return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, config.BackendLocal, err)
+		}
+		sourceBundle, err := bundle.Validate(ctx, sourceBackend, "")
+		if err != nil {
+			closeBackend(sourceBackend)
+			return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s validate source bundle: %w", pipeline.ID, config.BackendLocal, err)
+		}
+		return sourceBackend, []bundle.Bundle{sourceBundle}, config.BackendLocal, nil
+	}
+
+	sourceBackend, err := backends.openSource(ctx, pipeline.Source)
+	if err != nil {
+		return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
+	}
+	bundles, err := bundle.Discover(ctx, sourceBackend, "")
+	if err != nil {
+		closeBackend(sourceBackend)
+		return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
+	}
+	return sourceBackend, bundles, pipeline.Source.Backend, nil
 }
 
 type closeableBackend interface {

internal/app/run_coordinator.go

@@ -0,0 +1,131 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type PipelineRunID string
+
+type PipelineRunStatus string
+
+const (
+	PipelineRunRunning   PipelineRunStatus = "running"
+	PipelineRunSucceeded PipelineRunStatus = "succeeded"
+	PipelineRunFailed    PipelineRunStatus = "failed"
+)
+
+type PipelineRunRecord struct {
+	ID         PipelineRunID     `json:"id"`
+	PipelineID string            `json:"pipeline_id"`
+	Status     PipelineRunStatus `json:"status"`
+	StartedAt  time.Time         `json:"started_at"`
+	FinishedAt *time.Time        `json:"finished_at,omitempty"`
+	Report     RunReport         `json:"report,omitempty"`
+	Error      string            `json:"error,omitempty"`
+}
+
+type DuplicatePipelineRunError struct {
+	PipelineID string
+	RunID      PipelineRunID
+}
+
+func (err DuplicatePipelineRunError) Error() string {
+	if err.RunID == "" {
+		return fmt.Sprintf("pipeline %q already has an active run", err.PipelineID)
+	}
+	return fmt.Sprintf("pipeline %q already has active run %s", err.PipelineID, err.RunID)
+}
+
+func IsDuplicatePipelineRun(err error) bool {
+	var duplicate DuplicatePipelineRunError
+	return errors.As(err, &duplicate)
+}
+
+type PipelineRunCoordinator struct {
+	ctx    context.Context
+	run    pipelineRunFunc
+	now    func() time.Time
+	mu     sync.Mutex
+	nextID uint64
+	active map[string]PipelineRunRecord
+}
+
+type pipelineRunFunc func(context.Context, RunPipelineOptions) (RunReport, error)
+
+func NewPipelineRunCoordinator(ctx context.Context) *PipelineRunCoordinator {
+	return newPipelineRunCoordinator(ctx, RunPipeline)
+}
+
+func newPipelineRunCoordinator(ctx context.Context, run pipelineRunFunc) *PipelineRunCoordinator {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	return &PipelineRunCoordinator{
+		ctx:    ctx,
+		run:    run,
+		now:    time.Now,
+		active: map[string]PipelineRunRecord{},
+	}
+}
+
+func (coordinator *PipelineRunCoordinator) RunPipeline(ctx context.Context, options RunPipelineOptions) (PipelineRunRecord, error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := ctx.Err(); err != nil {
+		return PipelineRunRecord{}, err
+	}
+	record, err := coordinator.admit(options.PipelineID)
+	if err != nil {
+		return PipelineRunRecord{}, err
+	}
+	defer coordinator.clear(options.PipelineID)
+
+	report, runErr := coordinator.run(coordinator.ctx, options)
+	record.Report = report
+	finishedAt := coordinator.now().UTC()
+	record.FinishedAt = &finishedAt
+	if runErr != nil {
+		record.Status = PipelineRunFailed
+		record.Error = runErr.Error()
+		return record, runErr
+	}
+	record.Status = PipelineRunSucceeded
+	return record, nil
+}
+
+func (coordinator *PipelineRunCoordinator) admit(pipelineID string) (PipelineRunRecord, error) {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	if active, ok := coordinator.active[pipelineID]; ok {
+		return PipelineRunRecord{}, DuplicatePipelineRunError{
+			PipelineID: pipelineID,
+			RunID:      active.ID,
+		}
+	}
+	coordinator.nextID++
+	record := PipelineRunRecord{
+		ID:         PipelineRunID(fmt.Sprintf("run-%016d", coordinator.nextID)),
+		PipelineID: pipelineID,
+		Status:     PipelineRunRunning,
+		StartedAt:  coordinator.now().UTC(),
+	}
+	coordinator.active[pipelineID] = record
+	return record, nil
+}
+
+func (coordinator *PipelineRunCoordinator) clear(pipelineID string) {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	delete(coordinator.active, pipelineID)
+}
+
+func (coordinator *PipelineRunCoordinator) activeCount() int {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	return len(coordinator.active)
+}

internal/app/run_coordinator_test.go

@@ -0,0 +1,223 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestPipelineRunCoordinatorRejectsDuplicateActiveRun(t *testing.T) {
+	started := make(chan struct{})
+	release := make(chan struct{})
+	var startedOnce sync.Once
+	coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
+		startedOnce.Do(func() {
+			close(started)
+		})
+		<-release
+		return RunReport{}, nil
+	})
+	firstResult := make(chan runCoordinatorTestResult, 1)
+
+	go func() {
+		record, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
+		firstResult <- runCoordinatorTestResult{record: record, err: err}
+	}()
+	waitForSignal(t, started, "first run to start")
+
+	_, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
+	if err == nil || !IsDuplicatePipelineRun(err) {
+		t.Fatalf("RunPipeline() error = %v, want duplicate active run", err)
+	}
+	close(release)
+	result := waitForRunResult(t, firstResult)
+	if result.err != nil {
+		t.Fatalf("first RunPipeline() error = %v", result.err)
+	}
+	if result.record.Status != PipelineRunSucceeded || result.record.ID == "" || result.record.FinishedAt == nil {
+		t.Fatalf("first record = %#v, want succeeded completed record", result.record)
+	}
+	if got := coordinator.activeCount(); got != 0 {
+		t.Fatalf("active count = %d, want 0", got)
+	}
+}
+
+func TestPipelineRunCoordinatorAllowsDifferentActivePipelines(t *testing.T) {
+	started := make(chan string, 2)
+	release := make(chan struct{})
+	coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
+		started <- options.PipelineID
+		<-release
+		return RunReport{}, nil
+	})
+	firstResult := make(chan runCoordinatorTestResult, 1)
+	secondResult := make(chan runCoordinatorTestResult, 1)
+
+	go func() {
+		record, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports-one"})
+		firstResult <- runCoordinatorTestResult{record: record, err: err}
+	}()
+	go func() {
+		record, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports-two"})
+		secondResult <- runCoordinatorTestResult{record: record, err: err}
+	}()
+	startedPipelines := map[string]bool{
+		waitForPipelineID(t, started): true,
+		waitForPipelineID(t, started): true,
+	}
+	if !startedPipelines["reports-one"] || !startedPipelines["reports-two"] {
+		t.Fatalf("started pipelines = %#v, want both requested pipelines", startedPipelines)
+	}
+	if got := coordinator.activeCount(); got != 2 {
+		t.Fatalf("active count = %d, want 2", got)
+	}
+
+	close(release)
+	first := waitForRunResult(t, firstResult)
+	second := waitForRunResult(t, secondResult)
+	if first.err != nil || second.err != nil {
+		t.Fatalf("RunPipeline() errors = %v, %v; want nil", first.err, second.err)
+	}
+	if first.record.ID == second.record.ID {
+		t.Fatalf("run IDs matched: %q", first.record.ID)
+	}
+	if got := coordinator.activeCount(); got != 0 {
+		t.Fatalf("active count = %d, want 0", got)
+	}
+}
+
+func TestPipelineRunCoordinatorClearsActiveRunAfterSuccess(t *testing.T) {
+	coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
+		return RunReport{DryRun: options.DryRun}, nil
+	})
+
+	first, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports", DryRun: true})
+	if err != nil {
+		t.Fatalf("first RunPipeline() error = %v", err)
+	}
+	second, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
+	if err != nil {
+		t.Fatalf("second RunPipeline() error = %v", err)
+	}
+	if first.Status != PipelineRunSucceeded || second.Status != PipelineRunSucceeded {
+		t.Fatalf("statuses = %s, %s; want succeeded", first.Status, second.Status)
+	}
+	if !first.Report.DryRun {
+		t.Fatalf("first report dry_run = false, want true")
+	}
+	if got := coordinator.activeCount(); got != 0 {
+		t.Fatalf("active count = %d, want 0", got)
+	}
+}
+
+func TestPipelineRunCoordinatorClearsActiveRunAfterFailure(t *testing.T) {
+	runError := errors.New("run failed")
+	attempt := 0
+	coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
+		attempt++
+		if attempt == 1 {
+			return RunReport{}, runError
+		}
+		return RunReport{}, nil
+	})
+
+	first, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
+	if !errors.Is(err, runError) {
+		t.Fatalf("first RunPipeline() error = %v, want run failure", err)
+	}
+	if first.Status != PipelineRunFailed || first.Error != runError.Error() || first.FinishedAt == nil {
+		t.Fatalf("first record = %#v, want failed completed record", first)
+	}
+	second, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
+	if err != nil {
+		t.Fatalf("second RunPipeline() error = %v", err)
+	}
+	if second.Status != PipelineRunSucceeded {
+		t.Fatalf("second status = %s, want succeeded", second.Status)
+	}
+	if got := coordinator.activeCount(); got != 0 {
+		t.Fatalf("active count = %d, want 0", got)
+	}
+}
+
+func TestPipelineRunCoordinatorClearsActiveRunAfterCancellation(t *testing.T) {
+	runContext, cancel := context.WithCancel(context.Background())
+	coordinator := newPipelineRunCoordinator(runContext, func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
+		return RunReport{}, ctx.Err()
+	})
+	cancel()
+
+	record, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("RunPipeline() error = %v, want context canceled", err)
+	}
+	if record.Status != PipelineRunFailed || record.Error != context.Canceled.Error() {
+		t.Fatalf("record = %#v, want failed cancellation record", record)
+	}
+	_, err = coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("second RunPipeline() error = %v, want context canceled", err)
+	}
+	if IsDuplicatePipelineRun(err) {
+		t.Fatalf("second RunPipeline() error = %v, want cancellation instead of duplicate", err)
+	}
+	if got := coordinator.activeCount(); got != 0 {
+		t.Fatalf("active count = %d, want 0", got)
+	}
+}
+
+func TestPipelineRunCoordinatorUnknownPipelineDoesNotRemainActive(t *testing.T) {
+	coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
+		return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
+	})
+
+	_, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "missing"})
+	if err == nil || !IsPipelineNotFound(err) {
+		t.Fatalf("RunPipeline() error = %v, want pipeline not found", err)
+	}
+	_, err = coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "missing"})
+	if err == nil || !IsPipelineNotFound(err) || IsDuplicatePipelineRun(err) {
+		t.Fatalf("second RunPipeline() error = %v, want pipeline not found without duplicate", err)
+	}
+	if got := coordinator.activeCount(); got != 0 {
+		t.Fatalf("active count = %d, want 0", got)
+	}
+}
+
+type runCoordinatorTestResult struct {
+	record PipelineRunRecord
+	err    error
+}
+
+func waitForSignal(t *testing.T, signal <-chan struct{}, name string) {
+	t.Helper()
+	select {
+	case <-signal:
+	case <-time.After(time.Second):
+		t.Fatalf("timed out waiting for %s", name)
+	}
+}
+
+func waitForPipelineID(t *testing.T, pipelineIDs <-chan string) string {
+	t.Helper()
+	select {
+	case pipelineID := <-pipelineIDs:
+		return pipelineID
+	case <-time.After(time.Second):
+		t.Fatalf("timed out waiting for pipeline start")
+		return ""
+	}
+}
+
+func waitForRunResult(t *testing.T, results <-chan runCoordinatorTestResult) runCoordinatorTestResult {
+	t.Helper()
+	select {
+	case result := <-results:
+		return result
+	case <-time.After(time.Second):
+		t.Fatalf("timed out waiting for run result")
+		return runCoordinatorTestResult{}
+	}
+}

internal/app/run_output.go

@@ -10,61 +10,125 @@ import (
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 )
 
-func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error) {
+func WriteRunReport(w io.Writer, format OutputFormat, report RunReport) error {
-	if w == nil {
+	if IsJSONOutput(format) {
-		return
+		return WriteJSONEnvelope(w, "run", len(report.OutputErrors) == 0, report.Warnings, report, report.OutputErrors)
 	}
-	if planErr != nil {
+	return writeRunReportText(w, report)
-		destinationID := plan.DestinationID
+}
+
+func writeRunReportText(w io.Writer, report RunReport) error {
+	if w == nil {
+		return nil
+	}
+	if err := writeWarnings(w, report.PreambleWarnings); err != nil {
+		return err
+	}
+	if _, err := fmt.Fprintf(w, "Configured pipelines: %d\n", len(report.Pipelines)); err != nil {
+		return err
+	}
+	for _, pipeline := range report.Pipelines {
+		if err := writeWarnings(w, pipeline.Warnings); err != nil {
+			return err
+		}
+		if _, err := fmt.Fprintf(w, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.SourceBackend, pipeline.BundleCount, destinationIDSummary(pipeline.Destinations)); err != nil {
+			return err
+		}
+		for _, event := range pipeline.events {
+			if event.warning != nil {
+				if err := writeWarnings(w, []OutputWarning{*event.warning}); err != nil {
+					return err
+				}
+				continue
+			}
+			if event.actionIndex < 0 || event.actionIndex >= len(report.Actions) {
+				continue
+			}
+			writeRunActionLine(w, report.Actions[event.actionIndex])
+		}
+	}
+	_, err := fmt.Fprintln(w, report.Summary.Line())
+	return err
+}
+
+func writeRunActionLine(w io.Writer, action RunActionRecord) {
+	if action.Action == "error" {
+		destinationID := action.DestinationID
 		if destinationID == "" {
 			destinationID = "unknown"
 		}
-		fmt.Fprintf(w, "  - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, pathMappingSummary(plan), planErr.Error())
+		fmt.Fprintf(w, "  - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", action.BundlePath, destinationID, action.Backend, pathMappingRecordSummary(action), action.Reason)
 		return
 	}
-	fmt.Fprintf(w, "  - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, pathMappingSummary(plan), plan.Action, outputSummary(plan.Outputs), plan.Reason)
+	fmt.Fprintf(w, "  - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, outputRecordSummary(action.Outputs), action.Reason)
 }
 
-func pathMappingSummary(plan publish.Plan) string {
+func pathMappingRecordSummary(action RunActionRecord) string {
-	if plan.PathMapping != config.PathMappingFixed {
+	if action.PathMapping != config.PathMappingFixed {
 		return ""
 	}
-	return fmt.Sprintf(" path_mapping=fixed target=%s", storage.DisplayPath(plan.DestinationBundlePath))
+	return fmt.Sprintf(" path_mapping=fixed target=%s", action.DestinationPath)
 }
 
-func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
+func outputRecordSummary(outputs []RunOutputRecord) string {
-	if w == nil {
-		return
-	}
-	fmt.Fprintf(w, "  - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, backend, err.Error())
-}
-
-func outputSummary(outputs []publish.Output) string {
 	if len(outputs) == 0 {
 		return "none"
 	}
 	paths := make([]string, 0, len(outputs))
 	for _, output := range outputs {
-		paths = append(paths, output.DestinationPath)
+		paths = append(paths, output.Path)
 	}
 	return strings.Join(paths, ",")
 }
 
-type runResult struct {
+func destinationIDSummary(ids []string) string {
-	DryRun    bool                `json:"dry_run"`
+	if len(ids) == 0 {
-	Pipelines []runPipelineResult `json:"pipelines"`
+		return "none"
-	Actions   []runActionResult   `json:"actions"`
+	}
-	Summary   runSummaryResult    `json:"summary"`
+	return strings.Join(ids, ",")
 }
 
-type runPipelineResult struct {
+type RunReport struct {
-	ID            string   `json:"id"`
+	DryRun           bool                 `json:"dry_run"`
-	SourceBackend string   `json:"source_backend"`
+	Pipelines        []RunPipelineSummary `json:"pipelines"`
-	BundleCount   int      `json:"bundle_count"`
+	Actions          []RunActionRecord    `json:"actions"`
-	Destinations  []string `json:"destinations"`
+	Summary          RunSummaryCounters   `json:"summary"`
+	Warnings         []OutputWarning      `json:"-"`
+	OutputErrors     []OutputError        `json:"-"`
+	PreambleWarnings []OutputWarning      `json:"-"`
 }
 
-type runActionResult struct {
+func (r *RunReport) addWarning(warning OutputWarning) {
+	r.Warnings = append(r.Warnings, warning)
+}
+
+func (r *RunReport) addWarnings(warnings []OutputWarning) {
+	r.Warnings = append(r.Warnings, warnings...)
+}
+
+type RunPipelineSummary struct {
+	ID            string          `json:"id"`
+	SourceBackend string          `json:"source_backend"`
+	BundleCount   int             `json:"bundle_count"`
+	Destinations  []string        `json:"destinations"`
+	Warnings      []OutputWarning `json:"-"`
+	events        []runPipelineEvent
+}
+
+type runPipelineEvent struct {
+	warning     *OutputWarning
+	actionIndex int
+}
+
+func warningEvent(warning OutputWarning) runPipelineEvent {
+	return runPipelineEvent{warning: &warning, actionIndex: -1}
+}
+
+func actionEvent(actionIndex int) runPipelineEvent {
+	return runPipelineEvent{actionIndex: actionIndex}
+}
+
+type RunActionRecord struct {
 	PipelineID      string            `json:"pipeline_id,omitempty"`
 	DestinationID   string            `json:"destination_id"`
 	Backend         string            `json:"backend"`
@@ -75,10 +139,10 @@ type runActionResult struct {
 	Action          string            `json:"action"`
 	PrimaryURL      string            `json:"primary_url,omitempty"`
 	Reason          string            `json:"reason,omitempty"`
-	Outputs         []runOutputResult `json:"outputs"`
+	Outputs         []RunOutputRecord `json:"outputs"`
 }
 
-type runOutputResult struct {
+type RunOutputRecord struct {
 	Path       string `json:"path"`
 	Kind       string `json:"kind"`
 	SourcePath string `json:"source_path,omitempty"`
@@ -88,13 +152,13 @@ type runOutputResult struct {
 	Size       int64  `json:"size"`
 }
 
-func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActionResult {
+func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActionRecord {
 	if planErr != nil {
 		destinationID := plan.DestinationID
 		if destinationID == "" {
 			destinationID = "unknown"
 		}
-		return runActionResult{
+		return RunActionRecord{
 			PipelineID:      plan.PipelineID,
 			DestinationID:   destinationID,
 			Backend:         backend,
@@ -105,10 +169,10 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActi
 			Action:          "error",
 			PrimaryURL:      plan.PrimaryURL,
 			Reason:          planErr.Error(),
-			Outputs:         []runOutputResult{},
+			Outputs:         []RunOutputRecord{},
 		}
 	}
-	return runActionResult{
+	return RunActionRecord{
 		PipelineID:      plan.PipelineID,
 		DestinationID:   plan.DestinationID,
 		Backend:         backend,
@@ -123,8 +187,8 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActi
 	}
 }
 
-func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) runActionResult {
+func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) RunActionRecord {
-	return runActionResult{
+	return RunActionRecord{
 		PipelineID:      pipelineID,
 		DestinationID:   destinationID,
 		Backend:         backend,
@@ -132,15 +196,15 @@ func errorAction(pipelineID, destinationID, backend, bundlePath string, err erro
 		DestinationPath: storage.DisplayPath(bundlePath),
 		Action:          "error",
 		Reason:          err.Error(),
-		Outputs:         []runOutputResult{},
+		Outputs:         []RunOutputRecord{},
 	}
 }
 
-func runOutputsFromPlan(outputs []publish.Output) []runOutputResult {
+func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
-	results := make([]runOutputResult, 0, len(outputs))
+	results := make([]RunOutputRecord, 0, len(outputs))
 	for _, output := range outputs {
 		stateOutput := output.StateOutputFile()
-		results = append(results, runOutputResult{
+		results = append(results, RunOutputRecord{
 			Path:       stateOutput.Path,
 			Kind:       stateOutput.Kind,
 			SourcePath: stateOutput.SourcePath,

internal/app/run_selection.go

@@ -3,7 +3,6 @@ package app
 import (
 	"fmt"
 	"sort"
-	"strings"
 
 	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
 	"gitea.maximumdirect.net/eric/distributor/internal/config"
@@ -78,14 +77,3 @@ func destinationIDs(destinations []config.Destination) []string {
 	}
 	return ids
 }
-
-func destinationSummary(destinations []config.Destination) string {
-	if len(destinations) == 0 {
-		return "none"
-	}
-	ids := make([]string, 0, len(destinations))
-	for _, destination := range destinations {
-		ids = append(ids, destination.ID)
-	}
-	return strings.Join(ids, ",")
-}

internal/app/run_summary.go

@@ -39,15 +39,7 @@ func (s *runSummary) recordFixedPath() {
 	s.fixedPath++
 }
 
-func (s runSummary) Line() string {
+type RunSummaryCounters struct {
-	status := "ok"
-	if s.failures > 0 {
-		status = "failed"
-	}
-	return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun, s.fixedPath)
-}
-
-type runSummaryResult struct {
 	Status       string `json:"status"`
 	Planned      int    `json:"planned"`
 	PublishNew   int    `json:"publish_new"`
@@ -59,12 +51,16 @@ type runSummaryResult struct {
 	FixedPath    int    `json:"fixed_path"`
 }
 
-func (s runSummary) Result() runSummaryResult {
+func (s RunSummaryCounters) Line() string {
+	return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.ReplaceOlder, s.ForceReplace, s.Skipped, s.Failed, s.DryRun, s.FixedPath)
+}
+
+func (s runSummary) Result() RunSummaryCounters {
 	status := "ok"
 	if s.failures > 0 {
 		status = "failed"
 	}
-	return runSummaryResult{
+	return RunSummaryCounters{
 		Status:       status,
 		Planned:      s.planned,
 		PublishNew:   s.publishNew,

internal/app/run_test.go

@@ -3,6 +3,7 @@ package app
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -220,6 +221,116 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
 	}
 }
 
+func TestRunPipelineWithLocalSourcePublishesConfiguredDestination(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+
+	report, err := RunPipelineWithLocalSource(context.Background(), RunPipelineWithLocalSourceOptions{
+		ConfigPath: writeUploadPipelineConfig(t, destinationRoot),
+		PipelineID: "reports",
+		SourceRoot: sourceRoot,
+	})
+	if err != nil {
+		t.Fatalf("RunPipelineWithLocalSource() error = %v", err)
+	}
+
+	if got, want := report.Summary.Status, "ok"; got != want {
+		t.Fatalf("report status = %q, want %q", got, want)
+	}
+	if got, want := len(report.Pipelines), 1; got != want {
+		t.Fatalf("pipeline count = %d, want %d", got, want)
+	}
+	if got, want := report.Pipelines[0].SourceBackend, config.BackendLocal; got != want {
+		t.Fatalf("source backend = %q, want %q", got, want)
+	}
+	if got, want := report.Pipelines[0].BundleCount, 1; got != want {
+		t.Fatalf("bundle count = %d, want %d", got, want)
+	}
+	if got, want := len(report.Actions), 1; got != want {
+		t.Fatalf("action count = %d, want %d", got, want)
+	}
+	if report.Actions[0].PipelineID != "reports" || report.Actions[0].DestinationID != "archive" {
+		t.Fatalf("action = %#v, want reports/archive action", report.Actions[0])
+	}
+	testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
+	testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
+}
+
+func TestRunPipelineWithLocalSourceValidatesBeforeDestinationWrites(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	writeJSONManifest(t, sourceRoot, testutil.ValidManifest(testutil.BundleOptions{}))
+
+	_, err := RunPipelineWithLocalSource(context.Background(), RunPipelineWithLocalSourceOptions{
+		ConfigPath: writeUploadPipelineConfig(t, destinationRoot),
+		PipelineID: "reports",
+		SourceRoot: sourceRoot,
+	})
+	if err == nil {
+		t.Fatal("RunPipelineWithLocalSource() error = nil, want validation error")
+	}
+	if !strings.Contains(err.Error(), "validate source bundle") {
+		t.Fatalf("RunPipelineWithLocalSource() error = %v, want source validation context", err)
+	}
+	entries, readErr := os.ReadDir(destinationRoot)
+	if readErr != nil {
+		t.Fatalf("ReadDir() error = %v", readErr)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("destination entries = %d, want no writes", len(entries))
+	}
+}
+
+func TestRunPipelineWithLocalSourcePublishesToRegisteredDestinationBackends(t *testing.T) {
+	sourceRoot := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	s3Destination := fake.New()
+	sshDestination := fake.New()
+	cfg := config.Config{
+		Pipelines: []config.Pipeline{{
+			ID: "reports",
+			Source: config.Backend{
+				Backend: config.BackendHTTPUpload,
+				Upload:  config.HTTPUpload{TokenEnv: "UPLOAD_TOKEN"},
+			},
+			Destinations: []config.Destination{
+				{
+					ID:       "object-archive",
+					Backend:  config.BackendS3,
+					Endpoint: "http://s3.test",
+					Bucket:   "destination-bucket",
+				},
+				{
+					ID:      "ssh-archive",
+					Backend: config.BackendSSH,
+					Host:    "ssh.test",
+					Path:    "/destination",
+				},
+			},
+		}},
+	}
+	config.ApplyDefaults(&cfg)
+	provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
+		"s3:destination-bucket": s3Destination,
+		"ssh:/destination":      sshDestination,
+	})
+
+	report, err := runPipelineConfigWithLocalSourceAndBackendFactory(context.Background(), cfg, RunPipelineWithLocalSourceOptions{
+		PipelineID: "reports",
+		SourceRoot: sourceRoot,
+	}, provider)
+	if err != nil {
+		t.Fatalf("runPipelineConfigWithLocalSourceAndBackendFactory() error = %v", err)
+	}
+
+	if got, want := len(report.Actions), 2; got != want {
+		t.Fatalf("action count = %d, want %d", got, want)
+	}
+	testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nSunny.\n")
+	testutil.AssertFakeFile(t, sshDestination, "summary.txt", "Summary\n")
+}
+
 func TestRunExplicitPreserveRelativePathMappingMatchesDefault(t *testing.T) {
 	sourceRoot := t.TempDir()
 	destinationRoot := t.TempDir()
@@ -718,6 +829,176 @@ func TestRunJSONIncludesGeneratedOutputMetadata(t *testing.T) {
 	}
 }
 
+func TestBuildRunReportIncludesStructuredDryRunResults(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	configPath := testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingFixed, "https://reports.example.com/latest", config.LinkPrimaryAuto, false, true, config.TransformModeIndex)
+	cfg, err := config.LoadFile(configPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+
+	report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment)
+	if err != nil {
+		t.Fatalf("buildRunReportWithBackendFactory() error = %v", err)
+	}
+	if !report.DryRun || report.Summary.Status != "ok" || !report.Summary.DryRun {
+		t.Fatalf("report dry-run/status = dry_run:%t summary:%#v, want ok dry-run", report.DryRun, report.Summary)
+	}
+	if got, want := len(report.Pipelines), 1; got != want {
+		t.Fatalf("pipeline count = %d, want %d", got, want)
+	}
+	pipeline := report.Pipelines[0]
+	if pipeline.ID != "reports" || pipeline.SourceBackend != config.BackendLocal || pipeline.BundleCount != 1 || strings.Join(pipeline.Destinations, ",") != "archive" {
+		t.Fatalf("pipeline summary = %#v, want reports/local bundle summary", pipeline)
+	}
+	if got, want := len(report.Warnings), 1; got != want {
+		t.Fatalf("warning count = %d, want %d", got, want)
+	}
+	if !strings.Contains(report.Warnings[0].Message, "path_mapping=fixed candidates=1 selected_bundle=.") {
+		t.Fatalf("warning = %#v, want fixed path selection", report.Warnings[0])
+	}
+	if got, want := len(report.Actions), 1; got != want {
+		t.Fatalf("action count = %d, want %d", got, want)
+	}
+	action := report.Actions[0]
+	if action.PipelineID != "reports" || action.DestinationID != "archive" || action.Action != "publish_new" || action.PrimaryURL != "https://reports.example.com/latest/" {
+		t.Fatalf("action = %#v, want publish_new with primary URL", action)
+	}
+	if action.PathMapping != config.PathMappingFixed || action.DestinationPath != "." {
+		t.Fatalf("action path mapping = %q destination path = %q, want fixed root", action.PathMapping, action.DestinationPath)
+	}
+	if got, want := len(action.Outputs), 1; got != want {
+		t.Fatalf("output count = %d, want %d", got, want)
+	}
+	output := action.Outputs[0]
+	if output.Path != "index.html" || output.Kind != state.OutputKindGenerated || output.SourcePath != "report.md" || output.Transform != "markdown_to_html" || output.URL != "https://reports.example.com/latest/" {
+		t.Fatalf("output = %#v, want generated index metadata", output)
+	}
+	if report.Summary.Planned != 1 || report.Summary.PublishNew != 1 || report.Summary.FixedPath != 1 || report.Summary.Failed != 0 {
+		t.Fatalf("summary = %#v, want publish_new fixed path counters", report.Summary)
+	}
+	if len(report.OutputErrors) != 0 {
+		t.Fatalf("output errors = %#v, want none", report.OutputErrors)
+	}
+}
+
+func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
+	sourceRoot := t.TempDir()
+	firstDestination := t.TempDir()
+	secondDestination := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+	if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
+		t.Fatalf("write unmanaged file: %v", err)
+	}
+	cfg, err := config.LoadFile(writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination))
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+
+	report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{}, newBackendFactoryWithEnvironment)
+	if err == nil || !IsPartialResultError(err) {
+		t.Fatalf("buildRunReportWithBackendFactory() error = %v, want partial result error", err)
+	}
+	if report.Summary.Status != "failed" || report.Summary.Planned != 1 || report.Summary.PublishNew != 1 || report.Summary.Failed != 1 {
+		t.Fatalf("summary = %#v, want one planned publish and one failure", report.Summary)
+	}
+	if got, want := len(report.Actions), 2; got != want {
+		t.Fatalf("action count = %d, want %d", got, want)
+	}
+	if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "error" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
+		t.Fatalf("first action = %#v, want archive-one error", report.Actions[0])
+	}
+	if report.Actions[1].DestinationID != "archive-two" || report.Actions[1].Action != "publish_new" {
+		t.Fatalf("second action = %#v, want archive-two publish_new", report.Actions[1])
+	}
+	if got, want := len(report.OutputErrors), 1; got != want {
+		t.Fatalf("output error count = %d, want %d", got, want)
+	}
+	outputError := report.OutputErrors[0]
+	if outputError.PipelineID != "reports" || outputError.DestinationID != "archive-one" || outputError.Backend != config.BackendLocal || outputError.BundlePath != "." || !strings.Contains(outputError.Message, "fail_unmanaged") {
+		t.Fatalf("output error = %#v, want archive-one unmanaged failure", outputError)
+	}
+}
+
+func TestRunPipelineRunsOnlyRequestedPipeline(t *testing.T) {
+	firstSource := t.TempDir()
+	secondSource := t.TempDir()
+	firstDestination := t.TempDir()
+	secondDestination := t.TempDir()
+	writeSourceBundle(t, firstSource, "", testBundleOptions{ID: "reports.one"})
+	writeSourceBundle(t, secondSource, "", testBundleOptions{ID: "reports.two"})
+	configPath := writeTwoPipelineConfig(t, firstSource, firstDestination, secondSource, secondDestination)
+	notifier := &recordingNotifier{}
+
+	report, err := RunPipeline(context.Background(), RunPipelineOptions{
+		ConfigPath: configPath,
+		PipelineID: "reports-one",
+		Notifier:   notifier,
+	})
+	if err != nil {
+		t.Fatalf("RunPipeline() error = %v", err)
+	}
+	if got, want := len(report.Pipelines), 1; got != want {
+		t.Fatalf("pipeline count = %d, want %d", got, want)
+	}
+	if report.Pipelines[0].ID != "reports-one" {
+		t.Fatalf("pipeline id = %q, want reports-one", report.Pipelines[0].ID)
+	}
+	if got, want := len(report.Actions), 1; got != want {
+		t.Fatalf("action count = %d, want %d", got, want)
+	}
+	if report.Actions[0].PipelineID != "reports-one" || report.Actions[0].Action != "publish_new" {
+		t.Fatalf("action = %#v, want reports-one publish_new", report.Actions[0])
+	}
+	if got, want := len(notifier.events), 1; got != want {
+		t.Fatalf("notification count = %d, want %d", got, want)
+	}
+	if notifier.events[0].PipelineID != "reports-one" {
+		t.Fatalf("notification pipeline = %q, want reports-one", notifier.events[0].PipelineID)
+	}
+	testutil.AssertFile(t, filepath.Join(firstDestination, "report.md"), "# Report\nSunny.\n")
+	if entries, err := os.ReadDir(secondDestination); err != nil || len(entries) != 0 {
+		t.Fatalf("second destination entries = %v err=%v, want empty", entries, err)
+	}
+}
+
+func TestRunPipelineUnknownIDReturnsNotFound(t *testing.T) {
+	sourceRoot := t.TempDir()
+	destinationRoot := t.TempDir()
+	writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
+
+	_, err := RunPipeline(context.Background(), RunPipelineOptions{
+		ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
+		PipelineID: "missing",
+	})
+	if err == nil || !IsPipelineNotFound(err) {
+		t.Fatalf("RunPipeline() error = %v, want pipeline not found", err)
+	}
+	if !strings.Contains(err.Error(), `pipeline "missing" not found`) {
+		t.Fatalf("RunPipeline() error = %v, want pipeline id in message", err)
+	}
+}
+
+func TestRunStillRunsAllConfiguredPipelines(t *testing.T) {
+	firstSource := t.TempDir()
+	secondSource := t.TempDir()
+	firstDestination := t.TempDir()
+	secondDestination := t.TempDir()
+	writeSourceBundle(t, firstSource, "", testBundleOptions{ID: "reports.one"})
+	writeSourceBundle(t, secondSource, "", testBundleOptions{ID: "reports.two"})
+
+	err := Run(context.Background(), RunOptions{
+		ConfigPath: writeTwoPipelineConfig(t, firstSource, firstDestination, secondSource, secondDestination),
+	})
+	if err != nil {
+		t.Fatalf("Run() error = %v", err)
+	}
+	testutil.AssertFile(t, filepath.Join(firstDestination, "report.md"), "# Report\nSunny.\n")
+	testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
+}
+
 func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
 	sourceRoot := t.TempDir()
 	destinationRoot := t.TempDir()
@@ -1346,6 +1627,44 @@ func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestina
 	return testutil.WriteFanoutLocalConfig(t, sourceRoot, firstDestination, secondDestination)
 }
 
+func writeUploadPipelineConfig(t *testing.T, destinationRoot string) string {
+	t.Helper()
+	return writeConfigFile(t, `
+pipelines:
+  - id: reports
+    source:
+      backend: http_upload
+      token_env: UPLOAD_TOKEN
+    destinations:
+      - id: archive
+        backend: local
+        path: `+destinationRoot+`
+`)
+}
+
+func writeTwoPipelineConfig(t *testing.T, firstSource, firstDestination, secondSource, secondDestination string) string {
+	t.Helper()
+	return writeConfigFile(t, `
+pipelines:
+  - id: reports-one
+    source:
+      backend: local
+      path: `+firstSource+`
+    destinations:
+      - id: archive
+        backend: local
+        path: `+firstDestination+`
+  - id: reports-two
+    source:
+      backend: local
+      path: `+secondSource+`
+    destinations:
+      - id: archive
+        backend: local
+        path: `+secondDestination+`
+`)
+}
+
 func writeConfigFile(t *testing.T, body string) string {
 	t.Helper()
 	path := filepath.Join(t.TempDir(), "config.yml")
@@ -1360,6 +1679,20 @@ func writeDestinationState(t *testing.T, root, relative string, manifest bundle.
 	testutil.WriteDestinationState(t, root, relative, manifest, testutil.DestinationStateOptions{})
 }
 
+func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
+	t.Helper()
+	if err := os.MkdirAll(root, 0o755); err != nil {
+		t.Fatalf("mkdir manifest root: %v", err)
+	}
+	data, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		t.Fatalf("marshal manifest: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(root, bundle.ManifestName), append(data, '\n'), 0o600); err != nil {
+		t.Fatalf("write manifest: %v", err)
+	}
+}
+
 func readStateFile(t *testing.T, path string) state.DistributorState {
 	t.Helper()
 	return testutil.ReadDestinationState(t, path)

internal/app/serve.go

@@ -0,0 +1,62 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+)
+
+type ServeOptions struct {
+	ConfigPath string
+}
+
+func Serve(ctx context.Context, options ServeOptions) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	configPath := options.ConfigPath
+	if configPath == "" {
+		configPath = config.DefaultConfigPath
+	}
+	cfg, err := config.LoadFile(configPath)
+	if err != nil {
+		return err
+	}
+	secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
+	if err != nil {
+		return err
+	}
+
+	handler, err := newUploadHTTPHandler(ctx, cfg, secretLoad.Environment)
+	if err != nil {
+		return err
+	}
+	listener, err := net.Listen("tcp", cfg.Server.HTTP.Bind)
+	if err != nil {
+		return fmt.Errorf("bind HTTP server %q: %w", cfg.Server.HTTP.Bind, err)
+	}
+	defer listener.Close()
+
+	server := &http.Server{Handler: handler}
+	shutdownDone := make(chan struct{})
+	go func() {
+		defer close(shutdownDone)
+		<-ctx.Done()
+		_ = server.Shutdown(context.Background())
+	}()
+
+	err = server.Serve(listener)
+	if errors.Is(err, http.ErrServerClosed) {
+		<-shutdownDone
+		return nil
+	}
+	return err
+}

internal/app/source_select.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"gitea.maximumdirect.net/eric/distributor/internal/bundle"
@@ -9,6 +10,19 @@ import (
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 )
 
+type PipelineNotFoundError struct {
+	ID string
+}
+
+func (e PipelineNotFoundError) Error() string {
+	return fmt.Sprintf("pipeline %q not found", e.ID)
+}
+
+func IsPipelineNotFound(err error) bool {
+	var notFound PipelineNotFoundError
+	return errors.As(err, &notFound)
+}
+
 type sourceCommandOptions struct {
 	CommandName string
 	Path        string
@@ -70,7 +84,7 @@ func selectSourceBundlesFromConfig(ctx context.Context, cfg config.Config, optio
 	}
 	pipeline, ok := findPipeline(cfg, options.PipelineID)
 	if !ok {
-		return sourceSelection{}, fmt.Errorf("pipeline %q not found", options.PipelineID)
+		return sourceSelection{}, PipelineNotFoundError{ID: options.PipelineID}
 	}
 	backends := provider(secretLoad.Environment)
 	sourceBackend, err := backends.openSource(ctx, pipeline.Source)

internal/app/upload_coordinator.go

@@ -0,0 +1,389 @@
+package app
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+)
+
+const DefaultUploadMaxFileCount = 4096
+
+type UploadRunID string
+
+type UploadStatus string
+
+const (
+	UploadStatusAccepted  UploadStatus = "accepted"
+	UploadStatusQueued    UploadStatus = "queued"
+	UploadStatusRunning   UploadStatus = "running"
+	UploadStatusSucceeded UploadStatus = "succeeded"
+	UploadStatusFailed    UploadStatus = "failed"
+	UploadStatusExpired   UploadStatus = "expired"
+)
+
+type UploadRunRecord struct {
+	ID         UploadRunID  `json:"run_id"`
+	PipelineID string       `json:"pipeline_id"`
+	Status     UploadStatus `json:"status"`
+	AcceptedAt time.Time    `json:"accepted_at"`
+	StartedAt  *time.Time   `json:"started_at,omitempty"`
+	FinishedAt *time.Time   `json:"finished_at,omitempty"`
+	Report     *RunReport   `json:"report,omitempty"`
+	Error      string       `json:"error,omitempty"`
+	StagedRoot string       `json:"-"`
+}
+
+type UploadRequest struct {
+	PipelineID   string
+	ContentType  string
+	Body         io.Reader
+	DryRun       bool
+	Force        bool
+	MaxFileCount int
+}
+
+type UploadQueueFullError struct {
+	QueueSize int
+}
+
+func (err UploadQueueFullError) Error() string {
+	return fmt.Sprintf("upload queue is full with capacity %d", err.QueueSize)
+}
+
+func IsUploadQueueFull(err error) bool {
+	var full UploadQueueFullError
+	return errors.As(err, &full)
+}
+
+type UploadCoordinator struct {
+	ctx          context.Context
+	cfg          config.Config
+	stage        uploadStageFunc
+	run          uploadRunFunc
+	now          func() time.Time
+	randomSuffix func() (string, error)
+	retention    time.Duration
+
+	mu             sync.Mutex
+	signal         chan struct{}
+	queueSize      int
+	maxConcurrency int
+	runningCount   int
+	activePipeline map[string]bool
+	pending        []*uploadJob
+	records        map[UploadRunID]UploadRunRecord
+}
+
+type uploadStageFunc func(context.Context, ingest.StageOptions) (ingest.StagedBundle, error)
+
+type uploadRunFunc func(context.Context, config.Config, RunPipelineWithLocalSourceOptions) (RunReport, error)
+
+type uploadJob struct {
+	recordID UploadRunID
+	request  UploadRequest
+	pipeline config.Pipeline
+}
+
+type uploadCoordinatorHooks struct {
+	stage        uploadStageFunc
+	run          uploadRunFunc
+	now          func() time.Time
+	randomSuffix func() (string, error)
+}
+
+func NewUploadCoordinator(ctx context.Context, cfg config.Config) *UploadCoordinator {
+	return newUploadCoordinator(ctx, cfg, uploadCoordinatorHooks{})
+}
+
+func newUploadCoordinator(ctx context.Context, cfg config.Config, hooks uploadCoordinatorHooks) *UploadCoordinator {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	config.ApplyDefaults(&cfg)
+	stage := hooks.stage
+	if stage == nil {
+		stage = ingest.StageArchive
+	}
+	run := hooks.run
+	if run == nil {
+		run = runPipelineConfigWithLocalSource
+	}
+	now := hooks.now
+	if now == nil {
+		now = time.Now
+	}
+	randomSuffix := hooks.randomSuffix
+	if randomSuffix == nil {
+		randomSuffix = randomRunIDSuffix
+	}
+	coordinator := &UploadCoordinator{
+		ctx:            ctx,
+		cfg:            cfg,
+		stage:          stage,
+		run:            run,
+		now:            now,
+		randomSuffix:   randomSuffix,
+		retention:      cfg.Server.HTTP.Retention.AsDuration(),
+		signal:         make(chan struct{}, 1),
+		queueSize:      cfg.Server.HTTP.QueueSize,
+		maxConcurrency: cfg.Server.HTTP.MaxConcurrency,
+		activePipeline: map[string]bool{},
+		records:        map[UploadRunID]UploadRunRecord{},
+	}
+	go coordinator.dispatchLoop()
+	return coordinator
+}
+
+func (coordinator *UploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := ctx.Err(); err != nil {
+		return UploadRunRecord{}, err
+	}
+	if request.Body == nil {
+		return UploadRunRecord{}, fmt.Errorf("upload body is required")
+	}
+	pipeline, ok := findPipeline(coordinator.cfg, request.PipelineID)
+	if !ok {
+		return UploadRunRecord{}, PipelineNotFoundError{ID: request.PipelineID}
+	}
+	if pipeline.Source.Backend != config.BackendHTTPUpload {
+		return UploadRunRecord{}, fmt.Errorf("pipeline %s source backend %s is not configured for uploads", pipeline.ID, pipeline.Source.Backend)
+	}
+	runID, err := coordinator.newRunID(pipeline.ID)
+	if err != nil {
+		return UploadRunRecord{}, err
+	}
+
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	coordinator.expireLocked(coordinator.now().UTC())
+	if len(coordinator.pending) >= coordinator.queueSize {
+		return UploadRunRecord{}, UploadQueueFullError{QueueSize: coordinator.queueSize}
+	}
+	record := UploadRunRecord{
+		ID:         runID,
+		PipelineID: pipeline.ID,
+		Status:     UploadStatusAccepted,
+		AcceptedAt: coordinator.now().UTC(),
+	}
+	coordinator.records[runID] = record
+	coordinator.pending = append(coordinator.pending, &uploadJob{
+		recordID: runID,
+		request:  request,
+		pipeline: pipeline,
+	})
+	coordinator.notify()
+	return record, nil
+}
+
+func (coordinator *UploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	coordinator.expireLocked(coordinator.now().UTC())
+	record, ok := coordinator.records[runID]
+	return record, ok
+}
+
+func (coordinator *UploadCoordinator) Expire() []UploadRunRecord {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	return coordinator.expireLocked(coordinator.now().UTC())
+}
+
+func (coordinator *UploadCoordinator) CanAccept() bool {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	coordinator.expireLocked(coordinator.now().UTC())
+	return len(coordinator.pending) < coordinator.queueSize
+}
+
+func (coordinator *UploadCoordinator) QueueDepth() int {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	return len(coordinator.pending)
+}
+
+func (coordinator *UploadCoordinator) RunningCount() int {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	return coordinator.runningCount
+}
+
+func (coordinator *UploadCoordinator) newRunID(pipelineID string) (UploadRunID, error) {
+	suffix, err := coordinator.randomSuffix()
+	if err != nil {
+		return "", err
+	}
+	timestamp := coordinator.now().UTC().Format("20060102T150405Z")
+	return UploadRunID(pipelineID + "." + timestamp + "." + suffix), nil
+}
+
+func (coordinator *UploadCoordinator) dispatchLoop() {
+	for {
+		select {
+		case <-coordinator.ctx.Done():
+			return
+		case <-coordinator.signal:
+			for coordinator.startNext() {
+			}
+		}
+	}
+}
+
+func (coordinator *UploadCoordinator) startNext() bool {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	if coordinator.runningCount >= coordinator.maxConcurrency {
+		coordinator.markPendingQueuedLocked()
+		return false
+	}
+	index := -1
+	for candidateIndex, job := range coordinator.pending {
+		if coordinator.activePipeline[job.pipeline.ID] {
+			record := coordinator.records[job.recordID]
+			if record.Status == UploadStatusAccepted {
+				record.Status = UploadStatusQueued
+				coordinator.records[job.recordID] = record
+			}
+			continue
+		}
+		index = candidateIndex
+		break
+	}
+	if index < 0 {
+		return false
+	}
+
+	job := coordinator.pending[index]
+	coordinator.pending = append(coordinator.pending[:index], coordinator.pending[index+1:]...)
+	now := coordinator.now().UTC()
+	record := coordinator.records[job.recordID]
+	record.Status = UploadStatusRunning
+	record.StartedAt = &now
+	coordinator.records[job.recordID] = record
+	coordinator.runningCount++
+	coordinator.activePipeline[job.pipeline.ID] = true
+
+	go coordinator.runJob(job)
+	return true
+}
+
+func (coordinator *UploadCoordinator) markPendingQueuedLocked() {
+	for _, job := range coordinator.pending {
+		record := coordinator.records[job.recordID]
+		if record.Status == UploadStatusAccepted {
+			record.Status = UploadStatusQueued
+			coordinator.records[job.recordID] = record
+		}
+	}
+}
+
+func (coordinator *UploadCoordinator) runJob(job *uploadJob) {
+	record := coordinator.currentRecord(job.recordID)
+	maxFileCount := job.request.MaxFileCount
+	if maxFileCount <= 0 {
+		maxFileCount = DefaultUploadMaxFileCount
+	}
+	staged, err := coordinator.stage(coordinator.ctx, ingest.StageOptions{
+		Body:                job.request.Body,
+		ContentType:         job.request.ContentType,
+		PipelineStagingPath: job.pipeline.Source.Upload.StagingPath,
+		RunID:               string(record.ID),
+		MaxUploadSize:       int64(*job.pipeline.Source.Upload.MaxUploadSize),
+		MaxExtractedSize:    int64(*job.pipeline.Source.Upload.MaxUploadSize),
+		MaxFileCount:        maxFileCount,
+	})
+	if err == nil {
+		coordinator.setStagedRoot(job.recordID, staged.Root)
+		var report RunReport
+		report, err = coordinator.run(coordinator.ctx, coordinator.cfg, RunPipelineWithLocalSourceOptions{
+			PipelineID: job.pipeline.ID,
+			SourceRoot: staged.Root,
+			DryRun:     job.request.DryRun,
+			Force:      job.request.Force,
+		})
+		coordinator.complete(job, &report, err)
+		return
+	}
+	coordinator.complete(job, nil, err)
+}
+
+func (coordinator *UploadCoordinator) currentRecord(runID UploadRunID) UploadRunRecord {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	return coordinator.records[runID]
+}
+
+func (coordinator *UploadCoordinator) setStagedRoot(runID UploadRunID, root string) {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	record := coordinator.records[runID]
+	record.StagedRoot = root
+	coordinator.records[runID] = record
+}
+
+func (coordinator *UploadCoordinator) complete(job *uploadJob, report *RunReport, runErr error) {
+	coordinator.mu.Lock()
+	defer coordinator.mu.Unlock()
+	record := coordinator.records[job.recordID]
+	finishedAt := coordinator.now().UTC()
+	record.FinishedAt = &finishedAt
+	record.Report = report
+	if runErr != nil {
+		record.Status = UploadStatusFailed
+		record.Error = runErr.Error()
+	} else {
+		record.Status = UploadStatusSucceeded
+	}
+	coordinator.records[job.recordID] = record
+	coordinator.runningCount--
+	delete(coordinator.activePipeline, job.pipeline.ID)
+	coordinator.notify()
+}
+
+func (coordinator *UploadCoordinator) expireLocked(now time.Time) []UploadRunRecord {
+	var expired []UploadRunRecord
+	for runID, record := range coordinator.records {
+		if record.FinishedAt == nil || record.Status == UploadStatusExpired {
+			continue
+		}
+		if now.Before(record.FinishedAt.Add(coordinator.retention)) {
+			continue
+		}
+		if record.StagedRoot != "" {
+			_ = os.RemoveAll(record.StagedRoot)
+		}
+		record.Status = UploadStatusExpired
+		record.Report = nil
+		record.Error = ""
+		expired = append(expired, record)
+		delete(coordinator.records, runID)
+	}
+	return expired
+}
+
+func (coordinator *UploadCoordinator) notify() {
+	select {
+	case coordinator.signal <- struct{}{}:
+	default:
+	}
+}
+
+func randomRunIDSuffix() (string, error) {
+	var data [4]byte
+	if _, err := rand.Read(data[:]); err != nil {
+		return "", fmt.Errorf("generate run id suffix: %w", err)
+	}
+	return hex.EncodeToString(data[:]), nil
+}

internal/app/upload_coordinator_test.go

@@ -0,0 +1,373 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+)
+
+func TestUploadCoordinatorGeneratesRunIDAndAcceptedStatus(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	clock := newUploadTestClock(time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC))
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"weather-daily"},
+	}), uploadCoordinatorHooks{
+		now:          clock.Now,
+		randomSuffix: uploadTestSuffixes("ab12cd34"),
+		stage:        successfulUploadStage,
+		run:          successfulUploadRun,
+	})
+
+	record, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "weather-daily",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("archive"),
+	})
+	if err != nil {
+		t.Fatalf("Submit() error = %v", err)
+	}
+	if got, want := record.ID, UploadRunID("weather-daily.20260603T120000Z.ab12cd34"); got != want {
+		t.Fatalf("run id = %q, want %q", got, want)
+	}
+	if got, want := record.Status, UploadStatusAccepted; got != want {
+		t.Fatalf("initial status = %q, want %q", got, want)
+	}
+	waitForUploadStatus(t, coordinator, record.ID, UploadStatusSucceeded)
+}
+
+func TestUploadCoordinatorRejectsFullQueueBeforeReadingBody(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	var reads atomic.Int64
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs:    []string{"reports"},
+		queueSize:      1,
+		maxConcurrency: 1,
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002", "00000003"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			<-release
+			return RunReport{}, nil
+		},
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("first"),
+	})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
+
+	second, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("second"),
+	})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusQueued)
+
+	_, err = coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body: readerFunc(func(data []byte) (int, error) {
+			reads.Add(1)
+			return 0, io.EOF
+		}),
+	})
+	if err == nil || !IsUploadQueueFull(err) {
+		t.Fatalf("third Submit() error = %v, want full queue", err)
+	}
+	if got := reads.Load(); got != 0 {
+		t.Fatalf("rejected body reads = %d, want 0", got)
+	}
+	close(release)
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
+}
+
+func TestUploadCoordinatorSerializesSamePipelineUploads(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs:    []string{"reports"},
+		queueSize:      4,
+		maxConcurrency: 2,
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			<-release
+			return RunReport{}, nil
+		},
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("first")})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	second, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("second")})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusQueued)
+	if got := coordinator.RunningCount(); got != 1 {
+		t.Fatalf("running count = %d, want 1", got)
+	}
+	close(release)
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
+}
+
+func TestUploadCoordinatorRunsDifferentPipelinesConcurrentlyUpToLimit(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs:    []string{"reports-one", "reports-two"},
+		queueSize:      4,
+		maxConcurrency: 2,
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			<-release
+			return RunReport{}, nil
+		},
+	})
+
+	first, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports-one", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("first")})
+	if err != nil {
+		t.Fatalf("first Submit() error = %v", err)
+	}
+	second, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports-two", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("second")})
+	if err != nil {
+		t.Fatalf("second Submit() error = %v", err)
+	}
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusRunning)
+	if got := coordinator.RunningCount(); got != 2 {
+		t.Fatalf("running count = %d, want 2", got)
+	}
+	close(release)
+	waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
+	waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
+}
+
+func TestUploadCoordinatorRecordsFailureDetails(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	runErr := errors.New("publish failed")
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+	}), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			return RunReport{DryRun: options.DryRun}, runErr
+		},
+	})
+
+	record, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("archive"),
+		DryRun:      true,
+	})
+	if err != nil {
+		t.Fatalf("Submit() error = %v", err)
+	}
+	failed := waitForUploadStatus(t, coordinator, record.ID, UploadStatusFailed)
+	if failed.Error != runErr.Error() {
+		t.Fatalf("error = %q, want %q", failed.Error, runErr.Error())
+	}
+	if failed.Report == nil || !failed.Report.DryRun {
+		t.Fatalf("report = %#v, want retained dry-run report", failed.Report)
+	}
+}
+
+func TestUploadCoordinatorExpiresCompletedRecordsAndStagingDirectories(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	clock := newUploadTestClock(time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC))
+	coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
+		pipelineIDs: []string{"reports"},
+		retention:   time.Second,
+	}), uploadCoordinatorHooks{
+		now:          clock.Now,
+		randomSuffix: uploadTestSuffixes("00000001"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			return RunReport{DryRun: true}, nil
+		},
+	})
+
+	record, err := coordinator.Submit(context.Background(), UploadRequest{
+		PipelineID:  "reports",
+		ContentType: ingest.ContentTypeTar,
+		Body:        strings.NewReader("archive"),
+	})
+	if err != nil {
+		t.Fatalf("Submit() error = %v", err)
+	}
+	succeeded := waitForUploadStatus(t, coordinator, record.ID, UploadStatusSucceeded)
+	if succeeded.Report == nil || !succeeded.Report.DryRun {
+		t.Fatalf("report = %#v, want retained dry-run report", succeeded.Report)
+	}
+	if _, err := os.Stat(succeeded.StagedRoot); err != nil {
+		t.Fatalf("staged root stat before expiry = %v", err)
+	}
+
+	clock.Advance(2 * time.Second)
+	expired := coordinator.Expire()
+	if got, want := len(expired), 1; got != want {
+		t.Fatalf("expired count = %d, want %d", got, want)
+	}
+	if expired[0].Status != UploadStatusExpired || expired[0].Report != nil || expired[0].Error != "" {
+		t.Fatalf("expired record = %#v, want expired without report/error", expired[0])
+	}
+	if _, ok := coordinator.Status(record.ID); ok {
+		t.Fatal("Status() ok = true after expiry, want removed status")
+	}
+	if _, err := os.Stat(succeeded.StagedRoot); !os.IsNotExist(err) {
+		t.Fatalf("staged root stat after expiry = %v, want not exist", err)
+	}
+}
+
+type readerFunc func([]byte) (int, error)
+
+func (fn readerFunc) Read(data []byte) (int, error) {
+	return fn(data)
+}
+
+func successfulUploadStage(ctx context.Context, opts ingest.StageOptions) (ingest.StagedBundle, error) {
+	root := filepath.Join(opts.PipelineStagingPath, opts.RunID)
+	if err := os.MkdirAll(root, 0o755); err != nil {
+		return ingest.StagedBundle{}, err
+	}
+	return ingest.StagedBundle{Root: root}, nil
+}
+
+func successfulUploadRun(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+	return RunReport{}, nil
+}
+
+type uploadCoordinatorConfigOptions struct {
+	pipelineIDs    []string
+	queueSize      int
+	maxConcurrency int
+	retention      time.Duration
+}
+
+func uploadCoordinatorConfig(t *testing.T, opts uploadCoordinatorConfigOptions) config.Config {
+	t.Helper()
+	queueSize := opts.queueSize
+	if queueSize == 0 {
+		queueSize = 4
+	}
+	maxConcurrency := opts.maxConcurrency
+	if maxConcurrency == 0 {
+		maxConcurrency = 1
+	}
+	retentionValue := opts.retention
+	if retentionValue == 0 {
+		retentionValue = time.Minute
+	}
+	retention := config.Duration(retentionValue)
+	maxUploadSize := config.ByteSize(1024)
+	cfg := config.Config{
+		Server: config.Server{HTTP: config.HTTPServer{
+			StagingRoot:    t.TempDir(),
+			MaxUploadSize:  &maxUploadSize,
+			QueueSize:      queueSize,
+			MaxConcurrency: maxConcurrency,
+			Retention:      &retention,
+		}},
+	}
+	for _, pipelineID := range opts.pipelineIDs {
+		cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
+			ID: pipelineID,
+			Source: config.Backend{
+				Backend: config.BackendHTTPUpload,
+				Upload:  config.HTTPUpload{TokenEnv: strings.ToUpper(strings.ReplaceAll(pipelineID, "-", "_")) + "_TOKEN"},
+			},
+			Destinations: []config.Destination{{
+				ID:      "archive",
+				Backend: config.BackendLocal,
+				Path:    t.TempDir(),
+			}},
+		})
+	}
+	return cfg
+}
+
+type uploadTestClock struct {
+	mu  sync.Mutex
+	now time.Time
+}
+
+func newUploadTestClock(now time.Time) *uploadTestClock {
+	return &uploadTestClock{now: now}
+}
+
+func (clock *uploadTestClock) Now() time.Time {
+	clock.mu.Lock()
+	defer clock.mu.Unlock()
+	return clock.now
+}
+
+func (clock *uploadTestClock) Advance(duration time.Duration) {
+	clock.mu.Lock()
+	defer clock.mu.Unlock()
+	clock.now = clock.now.Add(duration)
+}
+
+func uploadTestSuffixes(values ...string) func() (string, error) {
+	var mu sync.Mutex
+	index := 0
+	return func() (string, error) {
+		mu.Lock()
+		defer mu.Unlock()
+		if index >= len(values) {
+			return fmt.Sprintf("%08d", index+1), nil
+		}
+		value := values[index]
+		index++
+		return value, nil
+	}
+}
+
+func waitForUploadStatus(t *testing.T, coordinator *UploadCoordinator, runID UploadRunID, status UploadStatus) UploadRunRecord {
+	t.Helper()
+	deadline := time.Now().Add(time.Second)
+	for time.Now().Before(deadline) {
+		record, ok := coordinator.Status(runID)
+		if ok && record.Status == status {
+			return record
+		}
+		time.Sleep(time.Millisecond)
+	}
+	record, ok := coordinator.Status(runID)
+	t.Fatalf("timed out waiting for status %s; latest ok=%t record=%#v", status, ok, record)
+	return UploadRunRecord{}
+}

internal/app/upload_http.go

@@ -0,0 +1,209 @@
+package app
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+)
+
+type uploadCoordinator interface {
+	CanAccept() bool
+	Submit(context.Context, UploadRequest) (UploadRunRecord, error)
+	Status(UploadRunID) (UploadRunRecord, bool)
+}
+
+type uploadHTTPHandler struct {
+	coordinator uploadCoordinator
+	tokens      map[string]string
+	limits      map[string]int64
+}
+
+type uploadAcceptedResponse struct {
+	RunID  UploadRunID  `json:"run_id"`
+	Status UploadStatus `json:"status"`
+}
+
+type httpErrorResponse struct {
+	Error string `json:"error"`
+}
+
+func newUploadHTTPHandler(ctx context.Context, cfg config.Config, environment config.Environment) (http.Handler, error) {
+	config.ApplyDefaults(&cfg)
+	tokens, limits, err := resolveUploadTokens(cfg, environment)
+	if err != nil {
+		return nil, err
+	}
+	return uploadHTTPHandler{
+		coordinator: NewUploadCoordinator(ctx, cfg),
+		tokens:      tokens,
+		limits:      limits,
+	}, nil
+}
+
+func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]string, map[string]int64, error) {
+	tokens := make(map[string]string)
+	limits := make(map[string]int64)
+	for _, pipeline := range cfg.Pipelines {
+		if pipeline.Source.Backend != config.BackendHTTPUpload {
+			continue
+		}
+		tokenName := pipeline.Source.Upload.TokenEnv
+		token, ok := environment.Lookup(tokenName)
+		if !ok {
+			return nil, nil, fmt.Errorf("upload token environment variable %s is not set", tokenName)
+		}
+		if token == "" {
+			return nil, nil, fmt.Errorf("upload token environment variable %s is empty", tokenName)
+		}
+		if existing, exists := tokens[token]; exists {
+			return nil, nil, fmt.Errorf("upload token environment variables for pipelines %s and %s resolve to the same value", existing, pipeline.ID)
+		}
+		tokens[token] = pipeline.ID
+		limits[pipeline.ID] = int64(*pipeline.Source.Upload.MaxUploadSize)
+	}
+	return tokens, limits, nil
+}
+
+func (handler uploadHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	switch {
+	case r.Method == http.MethodGet && r.URL.Path == "/healthz":
+		handler.handleHealth(w)
+	case r.Method == http.MethodPost && r.URL.Path == "/upload":
+		handler.handleUpload(w, r)
+	case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/runs/"):
+		handler.handleRunStatus(w, r)
+	default:
+		writeHTTPError(w, http.StatusNotFound, "not found")
+	}
+}
+
+func (handler uploadHTTPHandler) handleHealth(w http.ResponseWriter) {
+	writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+}
+
+func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Query().Has("pipeline") || r.URL.Query().Has("pipeline_id") {
+		writeHTTPError(w, http.StatusBadRequest, "pipeline id is not accepted")
+		return
+	}
+	pipelineID, ok := handler.authenticate(r.Header.Get("Authorization"))
+	if !ok {
+		writeHTTPError(w, http.StatusUnauthorized, "unauthorized")
+		return
+	}
+	contentType := r.Header.Get("Content-Type")
+	if !supportedUploadContentType(contentType) {
+		writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
+		return
+	}
+	if !handler.coordinator.CanAccept() {
+		writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
+		return
+	}
+	body, err := readUploadBody(r.Body, handler.limits[pipelineID])
+	if err != nil {
+		if errors.Is(err, ingest.ErrUploadTooLarge) {
+			writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
+			return
+		}
+		writeHTTPError(w, http.StatusBadRequest, "read upload body failed")
+		return
+	}
+	record, err := handler.coordinator.Submit(r.Context(), UploadRequest{
+		PipelineID:  pipelineID,
+		ContentType: contentType,
+		Body:        bytes.NewReader(body),
+	})
+	if err != nil {
+		writeUploadSubmitError(w, err)
+		return
+	}
+	writeJSON(w, http.StatusAccepted, uploadAcceptedResponse{
+		RunID:  record.ID,
+		Status: UploadStatusAccepted,
+	})
+}
+
+func (handler uploadHTTPHandler) handleRunStatus(w http.ResponseWriter, r *http.Request) {
+	rawRunID := strings.TrimPrefix(r.URL.Path, "/runs/")
+	if rawRunID == "" || strings.Contains(rawRunID, "/") {
+		writeHTTPError(w, http.StatusNotFound, "not found")
+		return
+	}
+	record, ok := handler.coordinator.Status(UploadRunID(rawRunID))
+	if !ok {
+		writeHTTPError(w, http.StatusNotFound, "run not found")
+		return
+	}
+	writeJSON(w, http.StatusOK, record)
+}
+
+func (handler uploadHTTPHandler) authenticate(header string) (string, bool) {
+	const prefix = "Bearer "
+	if !strings.HasPrefix(header, prefix) {
+		return "", false
+	}
+	token := strings.TrimSpace(strings.TrimPrefix(header, prefix))
+	if token == "" {
+		return "", false
+	}
+	pipelineID, ok := handler.tokens[token]
+	return pipelineID, ok
+}
+
+func supportedUploadContentType(contentType string) bool {
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		mediaType = contentType
+	}
+	switch mediaType {
+	case ingest.ContentTypeTar, ingest.ContentTypeGzip, ingest.ContentTypeXGzip:
+		return true
+	default:
+		return false
+	}
+}
+
+func readUploadBody(body io.Reader, maxSize int64) ([]byte, error) {
+	limited := &io.LimitedReader{R: body, N: maxSize + 1}
+	data, err := io.ReadAll(limited)
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(data)) > maxSize {
+		return nil, ingest.ErrUploadTooLarge
+	}
+	return data, nil
+}
+
+func writeUploadSubmitError(w http.ResponseWriter, err error) {
+	switch {
+	case IsUploadQueueFull(err):
+		writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
+	case errors.Is(err, ingest.ErrUploadTooLarge):
+		writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
+	case errors.Is(err, ingest.ErrUnsupportedContentType):
+		writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
+	default:
+		writeHTTPError(w, http.StatusBadRequest, "upload rejected")
+	}
+}
+
+func writeHTTPError(w http.ResponseWriter, status int, message string) {
+	writeJSON(w, status, httpErrorResponse{Error: message})
+}
+
+func writeJSON(w http.ResponseWriter, status int, value any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(value)
+}

internal/app/upload_http_integration_test.go

@@ -0,0 +1,419 @@
+package app
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/ingest"
+	"gitea.maximumdirect.net/eric/distributor/internal/storage"
+	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
+)
+
+func TestHTTPUploadPublishesTarAndGzipFanout(t *testing.T) {
+	tests := []struct {
+		name        string
+		compressed  bool
+		contentType string
+	}{
+		{name: "tar", contentType: ingest.ContentTypeTar},
+		{name: "gzip", compressed: true, contentType: ingest.ContentTypeGzip},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			firstDestination := t.TempDir()
+			secondDestination := t.TempDir()
+			cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+				id:           "reports",
+				tokenEnv:     "REPORTS_TOKEN",
+				stagingPath:  filepath.Join(t.TempDir(), "reports"),
+				destinations: []string{firstDestination, secondDestination},
+			}}, 4, 1)
+			handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
+				"REPORTS_TOKEN": "reports-secret",
+			}))
+			if err != nil {
+				t.Fatalf("newUploadHTTPHandler() error = %v", err)
+			}
+			server := httptest.NewServer(handler)
+			defer server.Close()
+
+			runID := submitHTTPUpload(t, server, "reports-secret", tt.contentType, bundleArchive(t, tt.compressed, testutil.BundleOptions{}))
+			record := waitForHTTPUploadStatus(t, server, runID, UploadStatusSucceeded)
+
+			if record.Report == nil {
+				t.Fatal("completed status report = nil, want run report")
+			}
+			if record.Report.Summary.Status != "ok" {
+				t.Fatalf("summary status = %q, want ok", record.Report.Summary.Status)
+			}
+			if got, want := len(record.Report.Actions), 2; got != want {
+				t.Fatalf("action count = %d, want %d", got, want)
+			}
+			assertPublishedBundle(t, firstDestination)
+			assertPublishedBundle(t, secondDestination)
+		})
+	}
+}
+
+func TestHTTPUploadInvalidArchiveFailsWithoutPublishing(t *testing.T) {
+	destination := t.TempDir()
+	cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+		id:           "reports",
+		tokenEnv:     "REPORTS_TOKEN",
+		stagingPath:  filepath.Join(t.TempDir(), "reports"),
+		destinations: []string{destination},
+	}}, 4, 1)
+	handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
+		"REPORTS_TOKEN": "reports-secret",
+	}))
+	if err != nil {
+		t.Fatalf("newUploadHTTPHandler() error = %v", err)
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	runID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("not a tar archive"))
+	record := waitForHTTPUploadStatus(t, server, runID, UploadStatusFailed)
+
+	if record.Error == "" {
+		t.Fatal("failed status error is empty")
+	}
+	if record.Report != nil {
+		t.Fatalf("failed staging report = %#v, want nil", record.Report)
+	}
+	assertDirectoryEmpty(t, destination)
+}
+
+func TestHTTPUploadSamePipelineRequestsSerialize(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	started := make(chan struct{}, 1)
+	coordinator := newUploadCoordinator(ctx, httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
+		id:           "reports",
+		tokenEnv:     "REPORTS_TOKEN",
+		stagingPath:  filepath.Join(t.TempDir(), "reports"),
+		destinations: []string{t.TempDir()},
+	}}, 4, 2), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			select {
+			case started <- struct{}{}:
+			default:
+			}
+			<-release
+			return RunReport{}, nil
+		},
+	})
+	handler := uploadHTTPHandler{
+		coordinator: coordinator,
+		tokens:      map[string]string{"reports-secret": "reports"},
+		limits:      map[string]int64{"reports": 1024},
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	firstRunID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("first"))
+	waitForRunStart(t, started)
+	first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusRunning)
+	secondRunID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("second"))
+	second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusQueued)
+
+	if first.PipelineID != "reports" || second.PipelineID != "reports" {
+		t.Fatalf("statuses = %#v %#v, want same pipeline", first, second)
+	}
+	if got := coordinator.RunningCount(); got != 1 {
+		t.Fatalf("running count = %d, want 1", got)
+	}
+	close(release)
+	waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
+	waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
+}
+
+func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	release := make(chan struct{})
+	started := make(chan string, 2)
+	coordinator := newUploadCoordinator(ctx, httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
+		{
+			id:           "reports-one",
+			tokenEnv:     "REPORTS_ONE_TOKEN",
+			stagingPath:  filepath.Join(t.TempDir(), "reports-one"),
+			destinations: []string{t.TempDir()},
+		},
+		{
+			id:           "reports-two",
+			tokenEnv:     "REPORTS_TWO_TOKEN",
+			stagingPath:  filepath.Join(t.TempDir(), "reports-two"),
+			destinations: []string{t.TempDir()},
+		},
+	}, 4, 2), uploadCoordinatorHooks{
+		randomSuffix: uploadTestSuffixes("00000001", "00000002"),
+		stage:        successfulUploadStage,
+		run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
+			started <- options.PipelineID
+			<-release
+			return RunReport{}, nil
+		},
+	})
+	handler := uploadHTTPHandler{
+		coordinator: coordinator,
+		tokens: map[string]string{
+			"one-secret": "reports-one",
+			"two-secret": "reports-two",
+		},
+		limits: map[string]int64{
+			"reports-one": 1024,
+			"reports-two": 1024,
+		},
+	}
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	firstRunID := submitHTTPUpload(t, server, "one-secret", ingest.ContentTypeTar, []byte("first"))
+	secondRunID := submitHTTPUpload(t, server, "two-secret", ingest.ContentTypeTar, []byte("second"))
+	waitForStartedPipelines(t, started, "reports-one", "reports-two")
+	waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusRunning)
+	waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusRunning)
+
+	if got := coordinator.RunningCount(); got != 2 {
+		t.Fatalf("running count = %d, want 2", got)
+	}
+	close(release)
+	waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
+	waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
+}
+
+type httpUploadPipelineSpec struct {
+	id           string
+	tokenEnv     string
+	stagingPath  string
+	destinations []string
+}
+
+func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpec, queueSize, maxConcurrency int) config.Config {
+	t.Helper()
+	size := config.ByteSize(1024 * 1024)
+	retention := config.Duration(time.Minute)
+	cfg := config.Config{
+		Server: config.Server{HTTP: config.HTTPServer{
+			Bind:           config.DefaultHTTPBind,
+			StagingRoot:    t.TempDir(),
+			MaxUploadSize:  &size,
+			QueueSize:      queueSize,
+			MaxConcurrency: maxConcurrency,
+			Retention:      &retention,
+		}},
+	}
+	for _, spec := range pipelines {
+		pipeline := config.Pipeline{
+			ID: spec.id,
+			Source: config.Backend{
+				Backend: config.BackendHTTPUpload,
+				Upload: config.HTTPUpload{
+					TokenEnv:      spec.tokenEnv,
+					StagingPath:   spec.stagingPath,
+					MaxUploadSize: &size,
+				},
+			},
+		}
+		for index, destination := range spec.destinations {
+			pipeline.Destinations = append(pipeline.Destinations, config.Destination{
+				ID:      fmt.Sprintf("archive-%d", index+1),
+				Backend: config.BackendLocal,
+				Path:    destination,
+				Publish: &config.PublishPolicy{Source: true},
+			})
+		}
+		cfg.Pipelines = append(cfg.Pipelines, pipeline)
+	}
+	config.ApplyDefaults(&cfg)
+	return cfg
+}
+
+func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) UploadRunID {
+	t.Helper()
+	request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("NewRequest() error = %v", err)
+	}
+	request.Header.Set("Authorization", "Bearer "+token)
+	request.Header.Set("Content-Type", contentType)
+	response, err := server.Client().Do(request)
+	if err != nil {
+		t.Fatalf("POST /upload error = %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusAccepted {
+		t.Fatalf("POST /upload status = %d, want %d", response.StatusCode, http.StatusAccepted)
+	}
+	var accepted uploadAcceptedResponse
+	if err := json.NewDecoder(response.Body).Decode(&accepted); err != nil {
+		t.Fatalf("decode accepted response: %v", err)
+	}
+	if accepted.RunID == "" || accepted.Status != UploadStatusAccepted {
+		t.Fatalf("accepted response = %#v, want run id and accepted status", accepted)
+	}
+	return accepted.RunID
+}
+
+func waitForHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID, status UploadStatus) UploadRunRecord {
+	t.Helper()
+	deadline := time.Now().Add(3 * time.Second)
+	var latest UploadRunRecord
+	var latestStatus int
+	for time.Now().Before(deadline) {
+		latest, latestStatus = getHTTPUploadStatus(t, server, runID)
+		if latestStatus == http.StatusOK && latest.Status == status {
+			return latest
+		}
+		time.Sleep(time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for status %s; latest HTTP status=%d record=%#v", status, latestStatus, latest)
+	return UploadRunRecord{}
+}
+
+func getHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID) (UploadRunRecord, int) {
+	t.Helper()
+	response, err := server.Client().Get(server.URL + "/runs/" + string(runID))
+	if err != nil {
+		t.Fatalf("GET /runs error = %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return UploadRunRecord{}, response.StatusCode
+	}
+	var record UploadRunRecord
+	if err := json.NewDecoder(response.Body).Decode(&record); err != nil {
+		t.Fatalf("decode run status: %v", err)
+	}
+	return record, response.StatusCode
+}
+
+func bundleArchive(t *testing.T, compressed bool, opts testutil.BundleOptions) []byte {
+	t.Helper()
+	root := t.TempDir()
+	testutil.WriteSourceBundle(t, root, "", opts)
+	return tarDirectory(t, root, compressed)
+}
+
+func tarDirectory(t *testing.T, root string, compressed bool) []byte {
+	t.Helper()
+	var output bytes.Buffer
+	var writer io.WriteCloser = nopWriteCloser{writer: &output}
+	if compressed {
+		gzipWriter := gzip.NewWriter(&output)
+		writer = gzipWriter
+	}
+	tarWriter := tar.NewWriter(writer)
+	if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		relative, err := filepath.Rel(root, filePath)
+		if err != nil {
+			return err
+		}
+		data, err := os.ReadFile(filePath)
+		if err != nil {
+			return err
+		}
+		header := &tar.Header{
+			Name: filepath.ToSlash(relative),
+			Mode: 0o600,
+			Size: int64(len(data)),
+		}
+		if err := tarWriter.WriteHeader(header); err != nil {
+			return err
+		}
+		if _, err := tarWriter.Write(data); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("walk bundle: %v", err)
+	}
+	if err := tarWriter.Close(); err != nil {
+		t.Fatalf("close tar: %v", err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatalf("close archive: %v", err)
+	}
+	return output.Bytes()
+}
+
+type nopWriteCloser struct {
+	writer io.Writer
+}
+
+func (writer nopWriteCloser) Write(data []byte) (int, error) {
+	return writer.writer.Write(data)
+}
+
+func (writer nopWriteCloser) Close() error {
+	return nil
+}
+
+func assertPublishedBundle(t *testing.T, destinationRoot string) {
+	t.Helper()
+	testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
+	testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
+	if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
+		t.Fatalf("destination state stat: %v", err)
+	}
+}
+
+func assertDirectoryEmpty(t *testing.T, root string) {
+	t.Helper()
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		t.Fatalf("ReadDir() error = %v", err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("directory %s has %d entries, want empty", root, len(entries))
+	}
+}
+
+func waitForRunStart(t *testing.T, started <-chan struct{}) {
+	t.Helper()
+	select {
+	case <-started:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for run start")
+	}
+}
+
+func waitForStartedPipelines(t *testing.T, started <-chan string, want ...string) {
+	t.Helper()
+	remaining := map[string]bool{}
+	for _, pipelineID := range want {
+		remaining[pipelineID] = true
+	}
+	deadline := time.After(time.Second)
+	for len(remaining) > 0 {
+		select {
+		case pipelineID := <-started:
+			delete(remaining, pipelineID)
+		case <-deadline:
+			t.Fatalf("timed out waiting for pipelines to start; remaining=%v", remaining)
+		}
+	}
+}

internal/app/upload_http_test.go

@@ -0,0 +1,331 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+)
+
+type fakeUploadCoordinator struct {
+	canAccept bool
+	submit    func(context.Context, UploadRequest) (UploadRunRecord, error)
+	status    func(UploadRunID) (UploadRunRecord, bool)
+}
+
+func (fake fakeUploadCoordinator) CanAccept() bool {
+	return fake.canAccept
+}
+
+func (fake fakeUploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) {
+	if fake.submit == nil {
+		return UploadRunRecord{}, errors.New("unexpected submit")
+	}
+	return fake.submit(ctx, request)
+}
+
+func (fake fakeUploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
+	if fake.status == nil {
+		return UploadRunRecord{}, false
+	}
+	return fake.status(runID)
+}
+
+func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) {
+	cfg := uploadHTTPTestConfig()
+
+	_, _, err := resolveUploadTokens(cfg, config.NewEnvironment(nil, func(string) (string, bool) {
+		return "", false
+	}))
+	if err == nil || !strings.Contains(err.Error(), "UPLOAD_TOKEN") {
+		t.Fatalf("resolveUploadTokens() error = %v, want missing UPLOAD_TOKEN", err)
+	}
+
+	cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
+		ID: "weekly",
+		Source: config.Backend{
+			Backend: config.BackendHTTPUpload,
+			Upload:  config.HTTPUpload{TokenEnv: "OTHER_UPLOAD_TOKEN"},
+		},
+		Destinations: cfg.Pipelines[0].Destinations,
+	})
+	config.ApplyDefaults(&cfg)
+	secret := "super-secret-token"
+	_, _, err = resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{
+		"UPLOAD_TOKEN":       secret,
+		"OTHER_UPLOAD_TOKEN": secret,
+	}))
+	if err == nil {
+		t.Fatal("resolveUploadTokens() error = nil, want duplicate token error")
+	}
+	if strings.Contains(err.Error(), secret) {
+		t.Fatalf("duplicate token error exposed secret value: %q", err)
+	}
+}
+
+func TestNewUploadHTTPHandlerAcceptsDefaultedConfig(t *testing.T) {
+	cfg := uploadHTTPTestConfig()
+	cfg.Server.HTTP.Bind = ""
+	cfg.Server.HTTP.StagingRoot = ""
+	cfg.Server.HTTP.MaxUploadSize = nil
+	cfg.Server.HTTP.QueueSize = 0
+	cfg.Server.HTTP.MaxConcurrency = 0
+	cfg.Server.HTTP.Retention = nil
+	cfg.Pipelines[0].Source.Upload.StagingPath = ""
+	cfg.Pipelines[0].Source.Upload.MaxUploadSize = nil
+
+	handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
+		"UPLOAD_TOKEN": "secret",
+	}))
+	if err != nil {
+		t.Fatalf("newUploadHTTPHandler() error = %v", err)
+	}
+	if handler == nil {
+		t.Fatal("newUploadHTTPHandler() = nil")
+	}
+}
+
+func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
+	var submitted UploadRequest
+	handler := uploadHTTPHandler{
+		coordinator: fakeUploadCoordinator{
+			canAccept: true,
+			submit: func(_ context.Context, request UploadRequest) (UploadRunRecord, error) {
+				submitted = request
+				body, err := io.ReadAll(request.Body)
+				if err != nil {
+					t.Fatalf("read submitted body: %v", err)
+				}
+				if string(body) != "archive" {
+					t.Fatalf("submitted body = %q, want archive", body)
+				}
+				return UploadRunRecord{ID: "reports.20260603T120000Z.abcdef12", Status: UploadStatusAccepted}, nil
+			},
+		},
+		tokens: map[string]string{"valid-token": "reports"},
+		limits: map[string]int64{"reports": 1024},
+	}
+	recorder := httptest.NewRecorder()
+	request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
+	request.Header.Set("Authorization", "Bearer valid-token")
+	request.Header.Set("Content-Type", "application/x-tar")
+
+	handler.ServeHTTP(recorder, request)
+
+	if recorder.Code != http.StatusAccepted {
+		t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusAccepted, recorder.Body.String())
+	}
+	if submitted.PipelineID != "reports" {
+		t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID)
+	}
+	var response uploadAcceptedResponse
+	if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+	if response.RunID != "reports.20260603T120000Z.abcdef12" || response.Status != UploadStatusAccepted {
+		t.Fatalf("response = %#v, want accepted run id", response)
+	}
+	if strings.Contains(recorder.Body.String(), "valid-token") {
+		t.Fatalf("response exposed token: %q", recorder.Body.String())
+	}
+}
+
+func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
+	handler := uploadHTTPHandler{
+		coordinator: fakeUploadCoordinator{canAccept: true},
+		tokens:      map[string]string{"valid-token": "reports"},
+		limits:      map[string]int64{"reports": 1024},
+	}
+
+	for _, authHeader := range []string{"", "Bearer wrong-token"} {
+		recorder := httptest.NewRecorder()
+		request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
+		request.Header.Set("Authorization", authHeader)
+		request.Header.Set("Content-Type", "application/x-tar")
+
+		handler.ServeHTTP(recorder, request)
+
+		if recorder.Code != http.StatusUnauthorized {
+			t.Fatalf("auth %q status = %d, want %d", authHeader, recorder.Code, http.StatusUnauthorized)
+		}
+		if strings.Contains(recorder.Body.String(), "valid-token") || strings.Contains(recorder.Body.String(), "wrong-token") {
+			t.Fatalf("unauthorized response exposed token: %q", recorder.Body.String())
+		}
+	}
+}
+
+func TestUploadHTTPHandlerRejectsUnsupportedOversizedFullQueueAndPipelineID(t *testing.T) {
+	tests := []struct {
+		name        string
+		canAccept   bool
+		url         string
+		contentType string
+		body        io.Reader
+		wantStatus  int
+	}{
+		{
+			name:        "unsupported content type",
+			canAccept:   true,
+			url:         "/upload",
+			contentType: "application/zip",
+			body:        strings.NewReader("archive"),
+			wantStatus:  http.StatusUnsupportedMediaType,
+		},
+		{
+			name:        "oversized",
+			canAccept:   true,
+			url:         "/upload",
+			contentType: "application/x-tar",
+			body:        strings.NewReader("too-large"),
+			wantStatus:  http.StatusRequestEntityTooLarge,
+		},
+		{
+			name:        "full queue",
+			canAccept:   false,
+			url:         "/upload",
+			contentType: "application/x-tar",
+			body:        &countingReader{reader: strings.NewReader("archive")},
+			wantStatus:  http.StatusServiceUnavailable,
+		},
+		{
+			name:        "submitted pipeline id",
+			canAccept:   true,
+			url:         "/upload?pipeline_id=reports",
+			contentType: "application/x-tar",
+			body:        strings.NewReader("archive"),
+			wantStatus:  http.StatusBadRequest,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			handler := uploadHTTPHandler{
+				coordinator: fakeUploadCoordinator{
+					canAccept: tt.canAccept,
+					submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
+						t.Fatal("Submit should not be called")
+						return UploadRunRecord{}, nil
+					},
+				},
+				tokens: map[string]string{"valid-token": "reports"},
+				limits: map[string]int64{"reports": 4},
+			}
+			recorder := httptest.NewRecorder()
+			request := httptest.NewRequest(http.MethodPost, tt.url, tt.body)
+			request.Header.Set("Authorization", "Bearer valid-token")
+			request.Header.Set("Content-Type", tt.contentType)
+
+			handler.ServeHTTP(recorder, request)
+
+			if recorder.Code != tt.wantStatus {
+				t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
+			}
+			if reader, ok := tt.body.(*countingReader); ok && reader.reads != 0 {
+				t.Fatalf("full queue read body %d time(s), want zero", reader.reads)
+			}
+		})
+	}
+}
+
+func TestUploadHTTPHandlerRunStatusAndHealth(t *testing.T) {
+	finishedAt := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
+	handler := uploadHTTPHandler{
+		coordinator: fakeUploadCoordinator{
+			canAccept: true,
+			status: func(runID UploadRunID) (UploadRunRecord, bool) {
+				if runID != "reports.20260603T120000Z.abcdef12" {
+					return UploadRunRecord{}, false
+				}
+				return UploadRunRecord{
+					ID:         runID,
+					PipelineID: "reports",
+					Status:     UploadStatusSucceeded,
+					FinishedAt: &finishedAt,
+				}, true
+			},
+		},
+		tokens: map[string]string{"valid-token": "reports"},
+		limits: map[string]int64{"reports": 1024},
+	}
+
+	recorder := httptest.NewRecorder()
+	handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+	if recorder.Code != http.StatusOK {
+		t.Fatalf("health status = %d, want %d", recorder.Code, http.StatusOK)
+	}
+
+	recorder = httptest.NewRecorder()
+	handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/reports.20260603T120000Z.abcdef12", nil))
+	if recorder.Code != http.StatusOK {
+		t.Fatalf("run status = %d, want %d; body = %q", recorder.Code, http.StatusOK, recorder.Body.String())
+	}
+	var record UploadRunRecord
+	if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
+		t.Fatalf("decode run status: %v", err)
+	}
+	if record.ID != "reports.20260603T120000Z.abcdef12" || record.Status != UploadStatusSucceeded {
+		t.Fatalf("record = %#v, want succeeded run status", record)
+	}
+
+	recorder = httptest.NewRecorder()
+	handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/unknown", nil))
+	if recorder.Code != http.StatusNotFound {
+		t.Fatalf("unknown run status = %d, want %d", recorder.Code, http.StatusNotFound)
+	}
+}
+
+type countingReader struct {
+	reader io.Reader
+	reads  int
+}
+
+func (reader *countingReader) Read(data []byte) (int, error) {
+	reader.reads++
+	return reader.reader.Read(data)
+}
+
+func uploadHTTPTestConfig() config.Config {
+	size := config.ByteSize(1024)
+	retention := config.Duration(24 * time.Hour)
+	cfg := config.Config{
+		Server: config.Server{HTTP: config.HTTPServer{
+			Bind:           config.DefaultHTTPBind,
+			StagingRoot:    "/tmp/distributor-test",
+			MaxUploadSize:  &size,
+			QueueSize:      2,
+			MaxConcurrency: 1,
+			Retention:      &retention,
+		}},
+		Pipelines: []config.Pipeline{{
+			ID: "reports",
+			Source: config.Backend{
+				Backend: config.BackendHTTPUpload,
+				Upload: config.HTTPUpload{
+					TokenEnv:      "UPLOAD_TOKEN",
+					StagingPath:   "/tmp/distributor-test/reports",
+					MaxUploadSize: &size,
+				},
+			},
+			Destinations: []config.Destination{{
+				ID:      "local",
+				Backend: config.BackendLocal,
+				Path:    "/tmp/distributor-output",
+				Publish: &config.PublishPolicy{Source: true},
+			}},
+		}},
+	}
+	config.ApplyDefaults(&cfg)
+	return cfg
+}
+
+func uploadHTTPTestEnvironment(values map[string]string) config.Environment {
+	return config.NewEnvironment(values, func(string) (string, bool) {
+		return "", false
+	})
+}

internal/cli/root.go

@@ -29,6 +29,8 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
 		return versionCommand(ctx, args[1:], stdout, stderr)
 	case "run":
 		return runCommand(ctx, args[1:], stdout, stderr)
+	case "serve":
+		return serveCommand(ctx, args[1:], stdout, stderr)
 	case "validate":
 		return validateCommand(ctx, args[1:], stdout, stderr)
 	case "inspect":
@@ -51,6 +53,7 @@ Usage:
 Commands:
   version    Print version information
   run        Run configured distribution pipelines
+  serve      Run the HTTP upload server
   validate   Validate a source bundle or bundle tree
   inspect    Inspect bundles or distributor state
   manifest   Create source bundle manifests

internal/cli/root_test.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"testing"
 
+	"gitea.maximumdirect.net/eric/distributor/internal/app"
 	"gitea.maximumdirect.net/eric/distributor/internal/storage"
 	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
 	producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
@@ -92,6 +93,34 @@ func TestExecuteVersionJSON(t *testing.T) {
 	}
 }
 
+func TestExecuteServeParsesConfig(t *testing.T) {
+	originalServeApp := serveApp
+	defer func() {
+		serveApp = originalServeApp
+	}()
+	var gotOptions app.ServeOptions
+	serveApp = func(_ context.Context, options app.ServeOptions) error {
+		gotOptions = options
+		return nil
+	}
+	var stdout, stderr bytes.Buffer
+
+	code := Execute(context.Background(), []string{"serve", "--config", "config.yml"}, &stdout, &stderr)
+
+	if code != exitOK {
+		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
+	}
+	if gotOptions.ConfigPath != "config.yml" {
+		t.Fatalf("ConfigPath = %q, want config.yml", gotOptions.ConfigPath)
+	}
+	if stdout.Len() != 0 {
+		t.Fatalf("stdout = %q, want empty", stdout.String())
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
 func TestExecuteRejectsInvalidFormat(t *testing.T) {
 	var stdout, stderr bytes.Buffer
 
@@ -603,8 +632,12 @@ func TestExecuteRunDryRun(t *testing.T) {
 	if code != exitOK {
 		t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
 	}
-	if !strings.Contains(stdout.String(), "action=publish_new") {
+	wantStdout := "Configured pipelines: 1\n" +
-		t.Fatalf("stdout = %q, want config summary", stdout.String())
+		"- pipeline=reports source=local bundles=1 destinations=archive\n" +
+		"  - bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt reason=\"destination state is absent\"\n" +
+		"Final status: ok planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true fixed_path=0\n"
+	if got := stdout.String(); got != wantStdout {
+		t.Fatalf("stdout = %q, want %q", got, wantStdout)
 	}
 	if stderr.Len() != 0 {
 		t.Fatalf("stderr = %q, want empty", stderr.String())
@@ -631,6 +664,14 @@ func TestExecuteRunJSONDryRun(t *testing.T) {
 	if result["dry_run"] != true {
 		t.Fatalf("result = %#v, want dry_run true", result)
 	}
+	pipelines, ok := result["pipelines"].([]any)
+	if !ok || len(pipelines) != 1 {
+		t.Fatalf("pipelines = %#v, want one pipeline", result["pipelines"])
+	}
+	pipeline, ok := pipelines[0].(map[string]any)
+	if !ok || pipeline["id"] != "reports" || pipeline["source_backend"] != "local" || pipeline["bundle_count"] != float64(1) {
+		t.Fatalf("pipeline = %#v, want reports/local summary", pipelines[0])
+	}
 	actions, ok := result["actions"].([]any)
 	if !ok || len(actions) != 1 {
 		t.Fatalf("actions = %#v, want one action", result["actions"])
@@ -639,6 +680,14 @@ func TestExecuteRunJSONDryRun(t *testing.T) {
 	if !ok || action["action"] != "publish_new" {
 		t.Fatalf("action = %#v, want publish_new", actions[0])
 	}
+	outputs, ok := action["outputs"].([]any)
+	if !ok || len(outputs) != 2 {
+		t.Fatalf("outputs = %#v, want source outputs", action["outputs"])
+	}
+	summary, ok := result["summary"].(map[string]any)
+	if !ok || summary["status"] != "ok" || summary["planned"] != float64(1) || summary["publish_new"] != float64(1) || summary["dry_run"] != true {
+		t.Fatalf("summary = %#v, want ok dry-run publish counters", result["summary"])
+	}
 	if stderr.Len() != 0 {
 		t.Fatalf("stderr = %q, want empty", stderr.String())
 	}

internal/cli/serve.go

@@ -0,0 +1,46 @@
+package cli
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/app"
+)
+
+var serveApp = app.Serve
+
+func serveCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+	if hasHelp(args) {
+		printServeHelp(stdout)
+		return exitOK
+	}
+
+	flags := flag.NewFlagSet("serve", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	configPath := flags.String("config", "", "path to config file")
+	if err := flags.Parse(args); err != nil {
+		return exitUsage
+	}
+	if rejectPositionalArgs(stderr, "serve", flags.Args()) {
+		return exitUsage
+	}
+
+	if err := serveApp(ctx, app.ServeOptions{ConfigPath: *configPath}); err != nil {
+		return fail(stderr, err)
+	}
+	return exitOK
+}
+
+func printServeHelp(w io.Writer) {
+	fmt.Fprint(w, `Usage:
+  distributor serve --config <path>
+
+Options:
+  --config <path>  Path to config file
+
+Serve loads configured HTTP upload sources, resolves upload tokens through the
+configured secret environment, and starts the HTTP upload API.
+`)
+}

internal/config/config.go

@@ -1,10 +1,24 @@
 package config
 
 type Config struct {
+	Server    Server     `yaml:"server"`
 	Secrets   Secrets    `yaml:"secrets"`
 	Pipelines []Pipeline `yaml:"pipelines"`
 }
 
+type Server struct {
+	HTTP HTTPServer `yaml:"http"`
+}
+
+type HTTPServer struct {
+	Bind           string    `yaml:"bind"`
+	StagingRoot    string    `yaml:"staging_root"`
+	MaxUploadSize  *ByteSize `yaml:"max_upload_size"`
+	QueueSize      int       `yaml:"queue_size"`
+	MaxConcurrency int       `yaml:"max_concurrency"`
+	Retention      *Duration `yaml:"retention"`
+}
+
 type Secrets struct {
 	Directory string `yaml:"directory"`
 }
@@ -50,6 +64,13 @@ type Backend struct {
 	ForcePath *bool       `yaml:"force_path_style"`
 	Creds     Credentials `yaml:"credentials"`
 	SSH       SSH         `yaml:",inline"`
+	Upload    HTTPUpload  `yaml:",inline"`
+}
+
+type HTTPUpload struct {
+	TokenEnv      string    `yaml:"token_env"`
+	StagingPath   string    `yaml:"staging_path"`
+	MaxUploadSize *ByteSize `yaml:"max_upload_size"`
 }
 
 type SSH struct {

internal/config/defaults.go

@@ -1,13 +1,19 @@
 package config
 
-import "gitea.maximumdirect.net/eric/distributor/internal/transform"
+import (
+	"path/filepath"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/transform"
+)
 
 const DefaultConfigPath = "/usr/local/etc/distributor/config.yml"
 
 const (
-	BackendLocal = "local"
+	BackendLocal      = "local"
-	BackendSSH   = "ssh"
+	BackendSSH        = "ssh"
-	BackendS3    = "s3"
+	BackendS3         = "s3"
+	BackendHTTPUpload = "http_upload"
 )
 
 const (
@@ -38,10 +44,23 @@ const (
 
 const DefaultS3Region = "us-east-1"
 
+const (
+	DefaultHTTPBind           = "127.0.0.1:8080"
+	DefaultHTTPStagingRoot    = "/var/spool/distributor"
+	DefaultHTTPMaxUploadSize  = ByteSize(20 * 1024 * 1024)
+	DefaultHTTPQueueSize      = 16
+	DefaultHTTPMaxConcurrency = 1
+	DefaultHTTPRetention      = Duration(24 * time.Hour)
+)
+
 func ApplyDefaults(cfg *Config) {
+	applyHTTPServerDefaults(&cfg.Server.HTTP)
 	for pipelineIndex := range cfg.Pipelines {
 		pipeline := &cfg.Pipelines[pipelineIndex]
 		applyBackendDefaults(&pipeline.Source)
+		if pipeline.Source.Backend == BackendHTTPUpload {
+			applyHTTPUploadDefaults(&pipeline.Source.Upload, pipeline.ID, cfg.Server.HTTP)
+		}
 		if pipeline.Validation.OnDigestMismatch == "" {
 			pipeline.Validation.OnDigestMismatch = ValidationActionFail
 		}
@@ -76,6 +95,44 @@ func ApplyDefaults(cfg *Config) {
 	}
 }
 
+func applyHTTPServerDefaults(server *HTTPServer) {
+	if server.Bind == "" {
+		server.Bind = DefaultHTTPBind
+	}
+	if server.StagingRoot == "" {
+		server.StagingRoot = DefaultHTTPStagingRoot
+	}
+	if server.MaxUploadSize == nil {
+		server.MaxUploadSize = byteSize(DefaultHTTPMaxUploadSize)
+	}
+	if server.QueueSize == 0 {
+		server.QueueSize = DefaultHTTPQueueSize
+	}
+	if server.MaxConcurrency == 0 {
+		server.MaxConcurrency = DefaultHTTPMaxConcurrency
+	}
+	if server.Retention == nil {
+		server.Retention = duration(DefaultHTTPRetention)
+	}
+}
+
+func applyHTTPUploadDefaults(upload *HTTPUpload, pipelineID string, server HTTPServer) {
+	if upload.StagingPath == "" && pipelineID != "" {
+		upload.StagingPath = filepath.Join(server.StagingRoot, pipelineID)
+	}
+	if upload.MaxUploadSize == nil && server.MaxUploadSize != nil {
+		upload.MaxUploadSize = byteSize(*server.MaxUploadSize)
+	}
+}
+
+func byteSize(value ByteSize) *ByteSize {
+	return &value
+}
+
+func duration(value Duration) *Duration {
+	return &value
+}
+
 func applyBackendDefaults(backend *Backend) {
 	if backend.Backend == BackendSSH {
 		if backend.Port == 0 {

internal/config/load_test.go

@@ -208,6 +208,123 @@ pipelines:
 	}
 }
 
+func TestLoadFileDefaultsHTTPServerConfig(t *testing.T) {
+	cfg := loadConfig(t, `
+pipelines:
+  - id: reports
+    source:
+      backend: local
+      path: /source
+    destinations:
+      - id: archive
+        backend: local
+        path: /archive
+`)
+
+	server := cfg.Server.HTTP
+	if got, want := server.Bind, DefaultHTTPBind; got != want {
+		t.Fatalf("server.http.bind = %q, want %q", got, want)
+	}
+	if got, want := server.StagingRoot, DefaultHTTPStagingRoot; got != want {
+		t.Fatalf("server.http.staging_root = %q, want %q", got, want)
+	}
+	if got, want := *server.MaxUploadSize, DefaultHTTPMaxUploadSize; got != want {
+		t.Fatalf("server.http.max_upload_size = %s, want %s", got, want)
+	}
+	if got, want := server.QueueSize, DefaultHTTPQueueSize; got != want {
+		t.Fatalf("server.http.queue_size = %d, want %d", got, want)
+	}
+	if got, want := server.MaxConcurrency, DefaultHTTPMaxConcurrency; got != want {
+		t.Fatalf("server.http.max_concurrency = %d, want %d", got, want)
+	}
+	if got, want := *server.Retention, DefaultHTTPRetention; got != want {
+		t.Fatalf("server.http.retention = %s, want %s", got, want)
+	}
+}
+
+func TestLoadFileAcceptsHTTPUploadSourceConfig(t *testing.T) {
+	cfg := loadConfig(t, `
+server:
+  http:
+    bind: 127.0.0.1:9090
+    staging_root: /srv/distributor/staging
+    max_upload_size: 64MB
+    queue_size: 32
+    max_concurrency: 2
+    retention: 48h
+pipelines:
+  - id: weather-daily
+    source:
+      backend: http_upload
+      token_env: WEATHER_DAILY_UPLOAD_TOKEN
+      staging_path: /srv/distributor/staging/weather-daily
+      max_upload_size: 32MB
+    destinations:
+      - id: archive
+        backend: local
+        path: /archive
+`)
+
+	server := cfg.Server.HTTP
+	if got, want := server.Bind, "127.0.0.1:9090"; got != want {
+		t.Fatalf("server.http.bind = %q, want %q", got, want)
+	}
+	if got, want := server.StagingRoot, "/srv/distributor/staging"; got != want {
+		t.Fatalf("server.http.staging_root = %q, want %q", got, want)
+	}
+	if got, want := *server.MaxUploadSize, ByteSize(64*1024*1024); got != want {
+		t.Fatalf("server.http.max_upload_size = %s, want %s", got, want)
+	}
+	if got, want := server.QueueSize, 32; got != want {
+		t.Fatalf("server.http.queue_size = %d, want %d", got, want)
+	}
+	if got, want := server.MaxConcurrency, 2; got != want {
+		t.Fatalf("server.http.max_concurrency = %d, want %d", got, want)
+	}
+	if got, want := server.Retention.String(), "48h0m0s"; got != want {
+		t.Fatalf("server.http.retention = %s, want %s", got, want)
+	}
+
+	source := cfg.Pipelines[0].Source
+	if got, want := source.Backend, BackendHTTPUpload; got != want {
+		t.Fatalf("source.backend = %q, want %q", got, want)
+	}
+	if got, want := source.Upload.TokenEnv, "WEATHER_DAILY_UPLOAD_TOKEN"; got != want {
+		t.Fatalf("source.token_env = %q, want %q", got, want)
+	}
+	if got, want := source.Upload.StagingPath, "/srv/distributor/staging/weather-daily"; got != want {
+		t.Fatalf("source.staging_path = %q, want %q", got, want)
+	}
+	if got, want := *source.Upload.MaxUploadSize, ByteSize(32*1024*1024); got != want {
+		t.Fatalf("source.max_upload_size = %s, want %s", got, want)
+	}
+}
+
+func TestLoadFileDefaultsHTTPUploadSourceConfig(t *testing.T) {
+	cfg := loadConfig(t, `
+server:
+  http:
+    max_upload_size: 12MB
+pipelines:
+  - id: weather-daily
+    source:
+      backend: http_upload
+      token_env: WEATHER_DAILY_UPLOAD_TOKEN
+    destinations:
+      - id: archive
+        backend: local
+        path: /archive
+`)
+
+	source := cfg.Pipelines[0].Source
+	if got, want := source.Upload.StagingPath, "/var/spool/distributor/weather-daily"; got != want {
+		t.Fatalf("source.staging_path = %q, want %q", got, want)
+	}
+	if got, want := *source.Upload.MaxUploadSize, ByteSize(12*1024*1024); got != want {
+		t.Fatalf("source.max_upload_size = %s, want %s", got, want)
+	}
+}
+
 func TestLoadFileValidBackendConfigs(t *testing.T) {
 	tests := map[string]string{
 		"local": `
@@ -396,6 +513,26 @@ func TestLoadFileRejectsInvalidS3Config(t *testing.T) {
 	}
 }
 
+func TestLoadFileRejectsInvalidHTTPUploadConfig(t *testing.T) {
+	tests := map[string]string{
+		"server size":             `server: {http: {max_upload_size: 20XB}}`,
+		"source size":             `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
+		"zero source size":        `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
+		"server duration":         `server: {http: {retention: forever}}`,
+		"zero server duration":    `server: {http: {retention: 0s}}`,
+		"missing token env":       `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
+		"destination http upload": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: ingest, backend: http_upload}]}]`,
+		"literal token":           `pipelines: [{id: reports, source: {backend: http_upload, token: secret, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
+		"unknown server field":    `server: {http: {surprise: true}}`,
+		"unknown source field":    `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
+	}
+	for name, body := range tests {
+		t.Run(name, func(t *testing.T) {
+			assertLoadError(t, body, "")
+		})
+	}
+}
+
 func TestLoadFileDefaultsSSHConfig(t *testing.T) {
 	cfg := loadConfig(t, `
 pipelines:
@@ -581,6 +718,7 @@ func TestExampleConfigsLoad(t *testing.T) {
 		"../../examples/local-index.yml",
 		"../../examples/fan-out.yml",
 		"../../examples/archive-and-latest.yml",
+		"../../examples/http-upload-local.yml",
 		"../../examples/ssh-destination.yml",
 		"../../examples/s3-destination.yml",
 	} {

internal/config/quantity.go

@@ -0,0 +1,117 @@
+package config
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+type ByteSize int64
+
+type Duration time.Duration
+
+func (size *ByteSize) UnmarshalYAML(value *yaml.Node) error {
+	if value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
+		return fmt.Errorf("size must be a string with B, KB, MB, or GB suffix")
+	}
+
+	var raw string
+	if err := value.Decode(&raw); err != nil {
+		return err
+	}
+	parsed, err := ParseByteSize(raw)
+	if err != nil {
+		return err
+	}
+	*size = parsed
+	return nil
+}
+
+func (size ByteSize) String() string {
+	value := int64(size)
+	if value == 0 {
+		return "0B"
+	}
+	units := []struct {
+		suffix     string
+		multiplier int64
+	}{
+		{suffix: "GB", multiplier: 1024 * 1024 * 1024},
+		{suffix: "MB", multiplier: 1024 * 1024},
+		{suffix: "KB", multiplier: 1024},
+		{suffix: "B", multiplier: 1},
+	}
+	for _, unit := range units {
+		if value%unit.multiplier == 0 {
+			return strconv.FormatInt(value/unit.multiplier, 10) + unit.suffix
+		}
+	}
+	return strconv.FormatInt(value, 10) + "B"
+}
+
+func ParseByteSize(raw string) (ByteSize, error) {
+	value := strings.TrimSpace(raw)
+	if value == "" {
+		return 0, fmt.Errorf("size is required")
+	}
+
+	units := []struct {
+		suffix     string
+		multiplier int64
+	}{
+		{suffix: "GB", multiplier: 1024 * 1024 * 1024},
+		{suffix: "MB", multiplier: 1024 * 1024},
+		{suffix: "KB", multiplier: 1024},
+		{suffix: "B", multiplier: 1},
+	}
+	for _, unit := range units {
+		number, ok := strings.CutSuffix(value, unit.suffix)
+		if !ok {
+			continue
+		}
+		if strings.TrimSpace(number) != number || number == "" {
+			return 0, fmt.Errorf("size must be an integer followed by B, KB, MB, or GB")
+		}
+		parsed, err := strconv.ParseInt(number, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("size must be an integer followed by B, KB, MB, or GB")
+		}
+		if parsed < 0 {
+			return 0, fmt.Errorf("size must be non-negative")
+		}
+		const maxInt64 = int64(1<<63 - 1)
+		if parsed > 0 && parsed > maxInt64/unit.multiplier {
+			return 0, fmt.Errorf("size is too large")
+		}
+		return ByteSize(parsed * unit.multiplier), nil
+	}
+	return 0, fmt.Errorf("size must use B, KB, MB, or GB suffix")
+}
+
+func (duration *Duration) UnmarshalYAML(value *yaml.Node) error {
+	if value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
+		return fmt.Errorf("duration must be a string duration")
+	}
+
+	var raw string
+	if err := value.Decode(&raw); err != nil {
+		return err
+	}
+	parsed, err := time.ParseDuration(raw)
+	if err != nil {
+		return fmt.Errorf("duration must be a valid duration: %w", err)
+	}
+	*duration = Duration(parsed)
+	return nil
+}
+
+func (duration Duration) String() string {
+	return time.Duration(duration).String()
+}
+
+func (duration Duration) AsDuration() time.Duration {
+	return time.Duration(duration)
+}

internal/config/validate.go

@@ -22,6 +22,8 @@ func (e ValidationErrors) Error() string {
 func Validate(cfg Config) error {
 	var errs ValidationErrors
 
+	errs = validateHTTPServer(errs, "server.http", cfg.Server.HTTP)
+
 	if len(cfg.Pipelines) == 0 {
 		errs = append(errs, "pipelines is required")
 	}
@@ -72,14 +74,56 @@ func Validate(cfg Config) error {
 	return nil
 }
 
+func validateHTTPServer(errs ValidationErrors, context string, server HTTPServer) ValidationErrors {
+	if server.Bind == "" {
+		errs = append(errs, context+".bind is required")
+	}
+	if server.StagingRoot == "" {
+		errs = append(errs, context+".staging_root is required")
+	}
+	if server.MaxUploadSize == nil || *server.MaxUploadSize <= 0 {
+		errs = append(errs, context+".max_upload_size must be greater than zero")
+	}
+	if server.QueueSize <= 0 {
+		errs = append(errs, context+".queue_size must be greater than zero")
+	}
+	if server.MaxConcurrency <= 0 {
+		errs = append(errs, context+".max_concurrency must be greater than zero")
+	}
+	if server.Retention == nil || *server.Retention <= 0 {
+		errs = append(errs, context+".retention must be greater than zero")
+	}
+	return errs
+}
+
 func validateSourceBackend(errs ValidationErrors, context string, backend Backend) ValidationErrors {
+	if backend.Backend == BackendHTTPUpload {
+		return validateHTTPUploadSource(errs, context, backend.Upload)
+	}
 	return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.Endpoint, backend.Bucket, backend.Prefix, backend.SSH.HostKeyPolicy, backend.Creds)
 }
 
 func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors {
+	if destination.Backend == BackendHTTPUpload {
+		errs = append(errs, context+".backend "+BackendHTTPUpload+" is only supported for sources")
+		return errs
+	}
 	return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.Endpoint, destination.Bucket, destination.Prefix, destination.SSH.HostKeyPolicy, destination.Creds)
 }
 
+func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTPUpload) ValidationErrors {
+	if upload.TokenEnv == "" {
+		errs = append(errs, context+".token_env is required for http_upload backend")
+	}
+	if upload.StagingPath == "" {
+		errs = append(errs, context+".staging_path is required for http_upload backend")
+	}
+	if upload.MaxUploadSize == nil || *upload.MaxUploadSize <= 0 {
+		errs = append(errs, context+".max_upload_size must be greater than zero")
+	}
+	return errs
+}
+
 func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, endpoint, bucket, prefix string, hostKeyPolicy HostKeyPolicy, creds Credentials) ValidationErrors {
 	switch backend {
 	case "":

internal/ingest/archive.go

@@ -0,0 +1,346 @@
+package ingest
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
+)
+
+const (
+	ContentTypeTar   = "application/x-tar"
+	ContentTypeGzip  = "application/gzip"
+	ContentTypeXGzip = "application/x-gzip"
+)
+
+var (
+	ErrUnsupportedContentType = errors.New("unsupported archive content type")
+	ErrUploadTooLarge         = errors.New("upload exceeds maximum size")
+	ErrExtractedTooLarge      = errors.New("extracted bundle exceeds maximum size")
+	ErrTooManyFiles           = errors.New("extracted bundle has too many files")
+	ErrUnsafeArchivePath      = errors.New("unsafe archive path")
+)
+
+type StageOptions struct {
+	Body                io.Reader
+	ContentType         string
+	PipelineStagingPath string
+	RunID               string
+	MaxUploadSize       int64
+	MaxExtractedSize    int64
+	MaxFileCount        int
+}
+
+type StagedBundle struct {
+	Root     string
+	Manifest sourcebundle.Manifest
+}
+
+func StageArchive(ctx context.Context, opts StageOptions) (StagedBundle, error) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	if err := validateStageOptions(opts); err != nil {
+		return StagedBundle{}, err
+	}
+	format, err := archiveFormat(opts.ContentType)
+	if err != nil {
+		return StagedBundle{}, err
+	}
+
+	if err := os.MkdirAll(opts.PipelineStagingPath, 0o755); err != nil {
+		return StagedBundle{}, fmt.Errorf("create pipeline staging path: %w", err)
+	}
+	tempDir, err := os.MkdirTemp(opts.PipelineStagingPath, "."+opts.RunID+"-")
+	if err != nil {
+		return StagedBundle{}, fmt.Errorf("create staging temp dir: %w", err)
+	}
+	cleanupTemp := true
+	defer func() {
+		if cleanupTemp {
+			_ = os.RemoveAll(tempDir)
+		}
+	}()
+
+	archivePath := filepath.Join(tempDir, "upload.archive")
+	if err := writeLimited(ctx, archivePath, opts.Body, opts.MaxUploadSize); err != nil {
+		return StagedBundle{}, err
+	}
+
+	extractRoot := filepath.Join(tempDir, "bundle")
+	if err := os.Mkdir(extractRoot, 0o755); err != nil {
+		return StagedBundle{}, fmt.Errorf("create extraction root: %w", err)
+	}
+	if err := extractArchive(ctx, archivePath, extractRoot, format, opts.MaxExtractedSize, opts.MaxFileCount); err != nil {
+		return StagedBundle{}, err
+	}
+
+	manifest, err := sourcebundle.LoadManifest(extractRoot)
+	if err != nil {
+		return StagedBundle{}, err
+	}
+	if err := sourcebundle.ValidateBundle(extractRoot, manifest); err != nil {
+		return StagedBundle{}, err
+	}
+
+	finalRoot := filepath.Join(opts.PipelineStagingPath, opts.RunID)
+	if err := os.Rename(extractRoot, finalRoot); err != nil {
+		return StagedBundle{}, fmt.Errorf("commit staged bundle: %w", err)
+	}
+	cleanupTemp = false
+	if err := os.RemoveAll(tempDir); err != nil {
+		return StagedBundle{}, fmt.Errorf("remove staging temp dir: %w", err)
+	}
+	return StagedBundle{
+		Root:     finalRoot,
+		Manifest: manifest,
+	}, nil
+}
+
+func validateStageOptions(opts StageOptions) error {
+	if opts.Body == nil {
+		return fmt.Errorf("body is required")
+	}
+	if opts.PipelineStagingPath == "" {
+		return fmt.Errorf("pipeline staging path is required")
+	}
+	if err := validateRunID(opts.RunID); err != nil {
+		return err
+	}
+	if opts.MaxUploadSize <= 0 {
+		return fmt.Errorf("max upload size must be greater than zero")
+	}
+	if opts.MaxExtractedSize <= 0 {
+		return fmt.Errorf("max extracted size must be greater than zero")
+	}
+	if opts.MaxFileCount <= 0 {
+		return fmt.Errorf("max file count must be greater than zero")
+	}
+	return nil
+}
+
+func validateRunID(value string) error {
+	if value == "" {
+		return fmt.Errorf("run id is required")
+	}
+	if value == "." || value == ".." || strings.ContainsAny(value, `/\`) {
+		return fmt.Errorf("run id must be a single filesystem path segment")
+	}
+	return nil
+}
+
+type archiveKind int
+
+const (
+	archiveKindTar archiveKind = iota + 1
+	archiveKindGzip
+)
+
+func archiveFormat(contentType string) (archiveKind, error) {
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		mediaType = contentType
+	}
+	switch mediaType {
+	case ContentTypeTar:
+		return archiveKindTar, nil
+	case ContentTypeGzip, ContentTypeXGzip:
+		return archiveKindGzip, nil
+	default:
+		return 0, ErrUnsupportedContentType
+	}
+}
+
+func writeLimited(ctx context.Context, destination string, body io.Reader, maxSize int64) error {
+	file, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if err != nil {
+		return fmt.Errorf("create upload archive: %w", err)
+	}
+	defer file.Close()
+
+	limited := &limitedReader{ctx: ctx, reader: body, limit: maxSize}
+	if _, err := io.Copy(file, limited); err != nil {
+		return err
+	}
+	if err := file.Close(); err != nil {
+		return fmt.Errorf("write upload archive: %w", err)
+	}
+	return nil
+}
+
+type limitedReader struct {
+	ctx    context.Context
+	reader io.Reader
+	limit  int64
+	read   int64
+}
+
+func (r *limitedReader) Read(data []byte) (int, error) {
+	if err := r.ctx.Err(); err != nil {
+		return 0, err
+	}
+	if r.read == r.limit {
+		var probe [1]byte
+		n, err := r.reader.Read(probe[:])
+		if n > 0 {
+			return 0, ErrUploadTooLarge
+		}
+		return 0, err
+	}
+	remaining := r.limit - r.read
+	if int64(len(data)) > remaining+1 {
+		data = data[:remaining+1]
+	}
+	n, err := r.reader.Read(data)
+	if r.read+int64(n) > r.limit {
+		allowed := int(r.limit - r.read)
+		r.read = r.limit
+		return allowed, ErrUploadTooLarge
+	}
+	r.read += int64(n)
+	return n, err
+}
+
+func extractArchive(ctx context.Context, archivePath, destination string, format archiveKind, maxExtractedSize int64, maxFileCount int) error {
+	file, err := os.Open(archivePath)
+	if err != nil {
+		return fmt.Errorf("open upload archive: %w", err)
+	}
+	defer file.Close()
+
+	var reader io.Reader = file
+	var gzipReader *gzip.Reader
+	if format == archiveKindGzip {
+		gzipReader, err = gzip.NewReader(file)
+		if err != nil {
+			return fmt.Errorf("open gzip archive: %w", err)
+		}
+		defer gzipReader.Close()
+		reader = gzipReader
+	}
+
+	extractor := archiveExtractor{
+		ctx:              ctx,
+		destination:      destination,
+		maxExtractedSize: maxExtractedSize,
+		maxFileCount:     maxFileCount,
+	}
+	if err := extractor.extract(tar.NewReader(reader)); err != nil {
+		return err
+	}
+	if extractor.rootManifestCount != 1 {
+		return fmt.Errorf("archive must contain exactly one root-level manifest.json")
+	}
+	return nil
+}
+
+type archiveExtractor struct {
+	ctx               context.Context
+	destination       string
+	maxExtractedSize  int64
+	maxFileCount      int
+	extractedSize     int64
+	fileCount         int
+	rootManifestCount int
+	seenFiles         map[string]struct{}
+}
+
+func (e *archiveExtractor) extract(reader *tar.Reader) error {
+	e.seenFiles = make(map[string]struct{})
+	for {
+		if err := e.ctx.Err(); err != nil {
+			return err
+		}
+		header, err := reader.Next()
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("read tar archive: %w", err)
+		}
+		name, err := cleanArchivePath(header.Name)
+		if err != nil {
+			return err
+		}
+		if path.Base(name) == sourcebundle.ManifestName {
+			if name != sourcebundle.ManifestName {
+				return fmt.Errorf("nested manifest %q is not allowed", name)
+			}
+			e.rootManifestCount++
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(filepath.Join(e.destination, filepath.FromSlash(name)), 0o755); err != nil {
+				return fmt.Errorf("create archive directory %q: %w", name, err)
+			}
+		case tar.TypeReg, tar.TypeRegA:
+			if err := e.extractFile(reader, name, header.Size); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("archive entry %q has unsupported type %c", name, header.Typeflag)
+		}
+	}
+}
+
+func (e *archiveExtractor) extractFile(reader *tar.Reader, name string, size int64) error {
+	if size < 0 {
+		return fmt.Errorf("archive entry %q has invalid size", name)
+	}
+	e.fileCount++
+	if e.fileCount > e.maxFileCount {
+		return ErrTooManyFiles
+	}
+	e.extractedSize += size
+	if e.extractedSize > e.maxExtractedSize {
+		return ErrExtractedTooLarge
+	}
+	if _, exists := e.seenFiles[name]; exists {
+		return fmt.Errorf("archive entry %q is duplicated", name)
+	}
+	e.seenFiles[name] = struct{}{}
+
+	fullPath := filepath.Join(e.destination, filepath.FromSlash(name))
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
+		return fmt.Errorf("create archive parent for %q: %w", name, err)
+	}
+	file, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
+	if err != nil {
+		return fmt.Errorf("create archive file %q: %w", name, err)
+	}
+	defer file.Close()
+
+	if _, err := io.CopyN(file, reader, size); err != nil {
+		return fmt.Errorf("extract archive file %q: %w", name, err)
+	}
+	if err := file.Close(); err != nil {
+		return fmt.Errorf("extract archive file %q: %w", name, err)
+	}
+	return nil
+}
+
+func cleanArchivePath(value string) (string, error) {
+	if value == "" || strings.Contains(value, `\`) || strings.HasPrefix(value, "/") {
+		return "", ErrUnsafeArchivePath
+	}
+	cleaned := path.Clean(value)
+	if cleaned != value {
+		return "", ErrUnsafeArchivePath
+	}
+	for _, segment := range strings.Split(value, "/") {
+		if segment == "" || segment == "." || segment == ".." {
+			return "", ErrUnsafeArchivePath
+		}
+	}
+	return value, nil
+}

internal/ingest/archive_test.go

@@ -0,0 +1,382 @@
+package ingest
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
+)
+
+func TestStageArchiveAcceptsTar(t *testing.T) {
+	archive := validArchive(t, false)
+	staged := stageArchive(t, archive, ContentTypeTar)
+
+	if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
+		t.Fatalf("manifest id = %q, want %q", got, want)
+	}
+	if got := readFile(t, staged.Root, "report.md"); got != "# Report\n" {
+		t.Fatalf("report = %q", got)
+	}
+}
+
+func TestStageArchiveAcceptsGzipTar(t *testing.T) {
+	archive := validArchive(t, true)
+	staged := stageArchive(t, archive, ContentTypeGzip+"; charset=binary")
+
+	if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
+		t.Fatalf("manifest id = %q, want %q", got, want)
+	}
+	if got := readFile(t, staged.Root, "summary.txt"); got != "Summary\n" {
+		t.Fatalf("summary = %q", got)
+	}
+}
+
+func TestStageArchiveRejectsUnsupportedContentType(t *testing.T) {
+	err := stageArchiveError(t, validArchive(t, false), "application/zip", nil)
+	if !errors.Is(err, ErrUnsupportedContentType) {
+		t.Fatalf("StageArchive() error = %v, want ErrUnsupportedContentType", err)
+	}
+}
+
+func TestStageArchiveEnforcesMaxUploadSize(t *testing.T) {
+	archive := validArchive(t, false)
+	err := stageArchiveError(t, archive, ContentTypeTar, func(opts *StageOptions) {
+		opts.MaxUploadSize = int64(len(archive) - 1)
+	})
+	if !errors.Is(err, ErrUploadTooLarge) {
+		t.Fatalf("StageArchive() error = %v, want ErrUploadTooLarge", err)
+	}
+}
+
+func TestStageArchiveEnforcesExtractionLimits(t *testing.T) {
+	archive := validArchive(t, false)
+	tests := map[string]struct {
+		mutate  func(*StageOptions)
+		wantErr error
+	}{
+		"size": {
+			mutate: func(opts *StageOptions) {
+				opts.MaxExtractedSize = 1
+			},
+			wantErr: ErrExtractedTooLarge,
+		},
+		"files": {
+			mutate: func(opts *StageOptions) {
+				opts.MaxFileCount = 1
+			},
+			wantErr: ErrTooManyFiles,
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			err := stageArchiveError(t, archive, ContentTypeTar, tt.mutate)
+			if !errors.Is(err, tt.wantErr) {
+				t.Fatalf("StageArchive() error = %v, want %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestStageArchiveRejectsUnsafeEntries(t *testing.T) {
+	tests := map[string][]tarEntry{
+		"absolute path": {
+			fileEntry("/report.md", "report"),
+		},
+		"path traversal": {
+			fileEntry("../report.md", "report"),
+		},
+		"backslash path": {
+			fileEntry(`nested\report.md`, "report"),
+		},
+		"symlink": {
+			{name: "link.md", typeflag: tar.TypeSymlink, linkname: "report.md"},
+		},
+		"hardlink": {
+			{name: "link.md", typeflag: tar.TypeLink, linkname: "report.md"},
+		},
+		"device": {
+			{name: "device", typeflag: tar.TypeChar},
+		},
+	}
+	for name, entries := range tests {
+		t.Run(name, func(t *testing.T) {
+			err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
+			if err == nil {
+				t.Fatal("StageArchive() error = nil, want error")
+			}
+		})
+	}
+}
+
+func TestStageArchiveRejectsBundleValidationFailures(t *testing.T) {
+	tests := map[string][]tarEntry{
+		"missing manifest": {
+			fileEntry("report.md", "report"),
+		},
+		"nested manifest": {
+			fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.nested", fileSpec{path: "report.md", body: "report"}))),
+			fileEntry("nested/manifest.json", "{}"),
+			fileEntry("report.md", "report"),
+		},
+		"missing listed file": {
+			fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.missing", fileSpec{path: "missing.md", body: "missing"}))),
+		},
+		"digest mismatch": {
+			fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.digest", fileSpec{path: "report.md", body: "expected"}))),
+			fileEntry("report.md", "actual"),
+		},
+		"non regular listed file": {
+			fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.directory", fileSpec{path: "report.md", body: "report"}))),
+			{name: "report.md", typeflag: tar.TypeDir},
+		},
+	}
+	for name, entries := range tests {
+		t.Run(name, func(t *testing.T) {
+			err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
+			if err == nil {
+				t.Fatal("StageArchive() error = nil, want error")
+			}
+		})
+	}
+}
+
+func TestStageArchiveCleansUpFailedExtraction(t *testing.T) {
+	stagingPath := filepath.Join(t.TempDir(), "staging")
+	archive := makeArchive(t, false, fileEntry("../report.md", "report"))
+	_, err := StageArchive(context.Background(), StageOptions{
+		Body:                bytes.NewReader(archive),
+		ContentType:         ContentTypeTar,
+		PipelineStagingPath: stagingPath,
+		RunID:               "reports.20260603T120000Z.abcd",
+		MaxUploadSize:       int64(len(archive)),
+		MaxExtractedSize:    1024 * 1024,
+		MaxFileCount:        10,
+	})
+	if err == nil {
+		t.Fatal("StageArchive() error = nil, want error")
+	}
+	entries, err := os.ReadDir(stagingPath)
+	if err != nil {
+		t.Fatalf("ReadDir() error = %v", err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("staging entries = %d, want cleanup", len(entries))
+	}
+}
+
+func stageArchive(t *testing.T, archive []byte, contentType string) StagedBundle {
+	t.Helper()
+	staged, err := StageArchive(context.Background(), defaultStageOptions(t, archive, contentType))
+	if err != nil {
+		t.Fatalf("StageArchive() error = %v", err)
+	}
+	return staged
+}
+
+func stageArchiveError(t *testing.T, archive []byte, contentType string, mutate func(*StageOptions)) error {
+	t.Helper()
+	opts := defaultStageOptions(t, archive, contentType)
+	if mutate != nil {
+		mutate(&opts)
+	}
+	_, err := StageArchive(context.Background(), opts)
+	if err == nil {
+		t.Fatal("StageArchive() error = nil, want error")
+	}
+	return err
+}
+
+func defaultStageOptions(t *testing.T, archive []byte, contentType string) StageOptions {
+	t.Helper()
+	return StageOptions{
+		Body:                bytes.NewReader(archive),
+		ContentType:         contentType,
+		PipelineStagingPath: filepath.Join(t.TempDir(), "staging"),
+		RunID:               "reports.20260603T120000Z.abcd",
+		MaxUploadSize:       int64(len(archive)),
+		MaxExtractedSize:    1024 * 1024,
+		MaxFileCount:        10,
+	}
+}
+
+func validArchive(t *testing.T, compressed bool) []byte {
+	t.Helper()
+	root := filepath.Join(t.TempDir(), "bundle")
+	sourceRoot := t.TempDir()
+	writeFile(t, sourceRoot, "report.md", "# Report\n")
+	writeFile(t, sourceRoot, "summary.txt", "Summary\n")
+	_, err := sourcebundle.WriteBundle(sourcebundle.WriteBundleOptions{
+		Root:    root,
+		ID:      "reports.ingest",
+		Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
+		Files: []sourcebundle.BundleFile{
+			{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
+			{SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "summary.txt"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("WriteBundle() error = %v", err)
+	}
+
+	var entries []tarEntry
+	if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		relative, err := filepath.Rel(root, filePath)
+		if err != nil {
+			return err
+		}
+		data, err := os.ReadFile(filePath)
+		if err != nil {
+			return err
+		}
+		entries = append(entries, fileEntry(filepath.ToSlash(relative), string(data)))
+		return nil
+	}); err != nil {
+		t.Fatalf("walk bundle: %v", err)
+	}
+	return makeArchive(t, compressed, entries...)
+}
+
+type tarEntry struct {
+	name     string
+	typeflag byte
+	body     []byte
+	linkname string
+}
+
+func fileEntry(name, body string) tarEntry {
+	return tarEntry{name: name, typeflag: tar.TypeReg, body: []byte(body)}
+}
+
+func makeArchive(t *testing.T, compressed bool, entries ...tarEntry) []byte {
+	t.Helper()
+	var output bytes.Buffer
+	var writer *tar.Writer
+	var gzipWriter *gzip.Writer
+	if compressed {
+		gzipWriter = gzip.NewWriter(&output)
+		writer = tar.NewWriter(gzipWriter)
+	} else {
+		writer = tar.NewWriter(&output)
+	}
+	for _, entry := range entries {
+		header := &tar.Header{
+			Name:     entry.name,
+			Typeflag: entry.typeflag,
+			Size:     int64(len(entry.body)),
+			Mode:     0o644,
+			Linkname: entry.linkname,
+		}
+		if entry.typeflag == tar.TypeDir {
+			header.Size = 0
+			header.Mode = 0o755
+		}
+		if err := writer.WriteHeader(header); err != nil {
+			t.Fatalf("WriteHeader(%q) error = %v", entry.name, err)
+		}
+		if len(entry.body) > 0 {
+			if _, err := writer.Write(entry.body); err != nil {
+				t.Fatalf("Write(%q) error = %v", entry.name, err)
+			}
+		}
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatalf("close tar writer: %v", err)
+	}
+	if gzipWriter != nil {
+		if err := gzipWriter.Close(); err != nil {
+			t.Fatalf("close gzip writer: %v", err)
+		}
+	}
+	return output.Bytes()
+}
+
+type fileSpec struct {
+	path string
+	body string
+}
+
+func manifestFor(id string, files ...fileSpec) sourcebundle.Manifest {
+	manifest := sourcebundle.Manifest{
+		SchemaVersion: sourcebundle.SchemaVersion,
+		ID:            id,
+		Created:       time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
+	}
+	for _, file := range files {
+		manifest.Files = append(manifest.Files, sourcebundle.ManifestFile{
+			Path:   file.path,
+			SHA256: sourcebundle.FileDigest([]byte(file.body)),
+			Size:   int64(len(file.body)),
+		})
+	}
+	manifest.Digest = sourcebundle.BundleDigest(manifest.Files)
+	return manifest
+}
+
+func manifestJSON(t *testing.T, manifest sourcebundle.Manifest) string {
+	t.Helper()
+	data, err := sourcebundle.MarshalManifest(manifest)
+	if err != nil {
+		t.Fatalf("MarshalManifest() error = %v", err)
+	}
+	return string(data)
+}
+
+func writeFile(t *testing.T, root, relative, body string) {
+	t.Helper()
+	fullPath := filepath.Join(root, filepath.FromSlash(relative))
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
+		t.Fatalf("MkdirAll() error = %v", err)
+	}
+	if err := os.WriteFile(fullPath, []byte(body), 0o644); err != nil {
+		t.Fatalf("WriteFile() error = %v", err)
+	}
+}
+
+func readFile(t *testing.T, root, relative string) string {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative)))
+	if err != nil {
+		t.Fatalf("ReadFile() error = %v", err)
+	}
+	return string(data)
+}
+
+func TestCleanArchivePath(t *testing.T) {
+	tests := map[string]bool{
+		"manifest.json":       true,
+		"nested/report.md":    true,
+		"":                    false,
+		"/absolute.md":        false,
+		"../escape.md":        false,
+		"nested/../report.md": false,
+		`nested\report.md`:    false,
+		"./report.md":         false,
+		"nested//report.md":   false,
+	}
+	for value, wantOK := range tests {
+		t.Run(strings.ReplaceAll(value, "/", "_"), func(t *testing.T) {
+			_, err := cleanArchivePath(value)
+			if wantOK && err != nil {
+				t.Fatalf("cleanArchivePath(%q) error = %v", value, err)
+			}
+			if !wantOK && err == nil {
+				t.Fatalf("cleanArchivePath(%q) error = nil, want error", value)
+			}
+		})
+	}
+}