Compare commits
15 Commits
v0.2.0
...
dc1f1f11f9
| Author | SHA1 | Date | |
|---|---|---|---|
| dc1f1f11f9 | |||
| 0f1ef9e622 | |||
| 6beef58dbf | |||
| f0c10210eb | |||
| f9436a7423 | |||
| 65dd22f974 | |||
| 35c5237dfc | |||
| 28eb5e07a0 | |||
| 22ce15c707 | |||
| 00677148e2 | |||
| 87fcd0277b | |||
| 7b2caf4c01 | |||
| 761a2f0bc2 | |||
| f236a8086a | |||
| 44df38e555 |
@@ -2,7 +2,10 @@
|
||||
|
||||
`distributor` validates manifested report bundles and publishes selected source or generated artifacts to configured destinations.
|
||||
|
||||
It is a local-first CLI with SSH/SFTP and S3-compatible storage support: source bundles can be read from local or remote storage, destinations can be local directories or remote paths, and Markdown files can be rendered to HTML sidecars or `index.html`.
|
||||
It is a local-first CLI with SSH/SFTP, S3-compatible storage, and HTTP upload
|
||||
support: source bundles can be read from local or remote storage, pushed to the
|
||||
upload API, published to local directories or remote paths, and rendered from
|
||||
Markdown to HTML sidecars or `index.html`.
|
||||
|
||||
Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build, write, parse, and validate complete local source bundles with the same manifest contract used by `distributor`.
|
||||
|
||||
|
||||
42
docs/cli.md
42
docs/cli.md
@@ -14,6 +14,7 @@ This discovers the example source bundle and publishes source files to `workspac
|
||||
distributor [--help]
|
||||
distributor version [--format text|json]
|
||||
distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
|
||||
distributor serve [--config <path>]
|
||||
distributor validate [--format text|json] <path>
|
||||
distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
|
||||
distributor inspect [--format text|json] <path>
|
||||
@@ -23,11 +24,12 @@ distributor manifest create <bundle-path> --id <bundle-id> [options]
|
||||
|
||||
- `version`: prints the application name and version. Development builds print `distributor dev`.
|
||||
- `run`: loads a YAML config, discovers source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary.
|
||||
- `serve`: loads a YAML config, resolves HTTP upload bearer tokens, and runs the HTTP upload API.
|
||||
- `validate`: validates a local source bundle directory, a local source bundle tree, or one configured pipeline source.
|
||||
- `inspect`: validates source bundles and prints normalized bundle metadata for a local path or one configured pipeline source.
|
||||
- `manifest create`: creates `manifest.json` for a local source bundle directory.
|
||||
|
||||
`validate` and `inspect` have two mutually exclusive modes: a local path shortcut, or configured source mode with `--config <path> --pipeline <id>`. Configured source mode opens only the selected pipeline source and supports configured `local`, `ssh`, and `s3` sources. It does not open destinations. `run` executes configured sources and destinations.
|
||||
`validate` and `inspect` have two mutually exclusive modes: a local path shortcut, or configured source mode with `--config <path> --pipeline <id>`. Configured source mode opens only the selected pipeline source and supports configured `local`, `ssh`, and `s3` sources. It does not open destinations. `run` executes configured `local`, `ssh`, and `s3` sources and destinations. `serve` executes configured `http_upload` sources through the upload API and normal destination fan-out.
|
||||
|
||||
## Flag reference
|
||||
|
||||
@@ -49,6 +51,10 @@ Output-producing subcommands:
|
||||
- `--dry-run`: load config, discover bundles, inspect destination state, print planned actions and final status, and do not write output files, destination state, or SSH `known_hosts` entries.
|
||||
- `--force`: allow explicit destructive replacement for supported conflict cases in this run only.
|
||||
|
||||
`serve` flags:
|
||||
|
||||
- `--config <path>`: config file to load. If omitted, `serve` uses `/usr/local/etc/distributor/config.yml`.
|
||||
|
||||
`validate` and `inspect` configured source flags:
|
||||
|
||||
- `--config <path>`: config file to load for source validation or inspection. Required in configured source mode.
|
||||
@@ -127,6 +133,40 @@ Publish the local HTML example:
|
||||
go run ./cmd/distributor run --config examples/local-html.yml
|
||||
```
|
||||
|
||||
Start the HTTP upload API:
|
||||
|
||||
```sh
|
||||
DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN=<token> \
|
||||
go run ./cmd/distributor serve --config examples/http-upload-local.yml
|
||||
```
|
||||
|
||||
Upload an archive to the configured `http_upload` pipeline associated with a bearer token:
|
||||
|
||||
```sh
|
||||
curl -X POST http://127.0.0.1:8080/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/gzip" \
|
||||
--data-binary @bundle.tar.gz
|
||||
```
|
||||
|
||||
The upload response is accepted asynchronously:
|
||||
|
||||
```json
|
||||
{"run_id":"reports.20260603T120000Z.abcdef12","status":"accepted"}
|
||||
```
|
||||
|
||||
Check upload status:
|
||||
|
||||
```sh
|
||||
curl http://127.0.0.1:8080/runs/<run-id>
|
||||
```
|
||||
|
||||
Check server readiness:
|
||||
|
||||
```sh
|
||||
curl http://127.0.0.1:8080/healthz
|
||||
```
|
||||
|
||||
Preview local fan-out publication:
|
||||
|
||||
```sh
|
||||
|
||||
131
docs/config.md
131
docs/config.md
@@ -2,15 +2,18 @@
|
||||
|
||||
## Config File Location
|
||||
|
||||
`distributor run --config <path>` loads the YAML config at the provided path.
|
||||
`distributor run --config <path>` and `distributor serve --config <path>` load
|
||||
the YAML config at the provided path.
|
||||
|
||||
If `--config` is omitted, `run` uses:
|
||||
If `--config` is omitted, both commands use:
|
||||
|
||||
```text
|
||||
/usr/local/etc/distributor/config.yml
|
||||
```
|
||||
|
||||
Config parsing rejects unknown YAML fields. The executable backends are `local`, `ssh`, and `s3`.
|
||||
Config parsing rejects unknown YAML fields. The executable `run` backends are
|
||||
`local`, `ssh`, and `s3`. The `serve` command executes `http_upload` sources
|
||||
through the HTTP upload API and normal destination fan-out.
|
||||
|
||||
## Minimal Local Config
|
||||
|
||||
@@ -31,6 +34,14 @@ This publishes source files only. It uses the default validation and transfer po
|
||||
## Production-Oriented Local Config
|
||||
|
||||
```yaml
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:8080
|
||||
staging_root: /var/spool/distributor
|
||||
max_upload_size: 20MB
|
||||
queue_size: 16
|
||||
max_concurrency: 1
|
||||
retention: 24h
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
@@ -52,6 +63,84 @@ pipelines:
|
||||
on_conflict: fail
|
||||
```
|
||||
|
||||
## HTTP Upload Source Configuration
|
||||
|
||||
HTTP upload sources are configured as pipeline sources only. They are not valid
|
||||
destination backends. `distributor serve` maps each configured upload token to
|
||||
exactly one `http_upload` pipeline.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:8080
|
||||
staging_root: /var/spool/distributor
|
||||
max_upload_size: 20MB
|
||||
queue_size: 16
|
||||
max_concurrency: 1
|
||||
retention: 24h
|
||||
pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
staging_path: /var/spool/distributor/weather-daily
|
||||
max_upload_size: 20MB
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /srv/reports/archive
|
||||
```
|
||||
|
||||
`source.token_env` is required and names the environment variable or `secrets.directory` file that provides the bearer token. Literal upload tokens are not supported in YAML.
|
||||
|
||||
`source.staging_path` is optional. When omitted, it defaults to `<server.http.staging_root>/<pipeline id>`.
|
||||
|
||||
`source.max_upload_size` is optional. When omitted, it defaults to `server.http.max_upload_size`.
|
||||
|
||||
The server resolves each `token_env` through the real process environment and
|
||||
the configured `secrets.directory` resolver. Startup fails if any configured
|
||||
upload token is missing, empty, or resolves to the same value as another upload
|
||||
pipeline. Token values are not read from YAML and are not printed in API
|
||||
responses.
|
||||
|
||||
## HTTP Upload API
|
||||
|
||||
`distributor serve` binds to `server.http.bind`, which defaults to
|
||||
`127.0.0.1:8080`.
|
||||
|
||||
Routes:
|
||||
|
||||
- `GET /healthz`: returns readiness status after config and upload tokens load.
|
||||
- `POST /upload`: accepts one tar or tar.gz source bundle archive.
|
||||
- `GET /runs/<run_id>`: returns an in-memory upload status record, or `404` if the run id is unknown or expired.
|
||||
|
||||
`POST /upload` authenticates with:
|
||||
|
||||
```text
|
||||
Authorization: Bearer <token>
|
||||
```
|
||||
|
||||
The token selects the configured `http_upload` pipeline. Producers do not send a
|
||||
pipeline id. Requests with a submitted `pipeline` or `pipeline_id` query value
|
||||
are rejected.
|
||||
|
||||
Accepted upload content types:
|
||||
|
||||
- `application/x-tar`
|
||||
- `application/gzip`
|
||||
- `application/x-gzip`
|
||||
|
||||
Accepted uploads return:
|
||||
|
||||
```json
|
||||
{"run_id":"<id>","status":"accepted"}
|
||||
```
|
||||
|
||||
The run id can be queried through `GET /runs/<run_id>` while the status record
|
||||
is retained in memory. Completed records expire after `server.http.retention`;
|
||||
expiration also removes committed staged bundle directories for completed
|
||||
uploads.
|
||||
|
||||
## HTML Publication
|
||||
|
||||
To publish generated sidecar HTML from Markdown files:
|
||||
@@ -143,6 +232,12 @@ Output URLs are built from `links.base_url`, the destination bundle path, and th
|
||||
|
||||
Top level:
|
||||
|
||||
- `server.http.bind`: optional HTTP bind address; defaults to `127.0.0.1:8080`.
|
||||
- `server.http.staging_root`: optional root for default HTTP upload staging paths; defaults to `/var/spool/distributor`.
|
||||
- `server.http.max_upload_size`: optional default upload size limit; defaults to `20MB`.
|
||||
- `server.http.queue_size`: optional HTTP upload admission queue size; defaults to `16`.
|
||||
- `server.http.max_concurrency`: optional HTTP upload worker concurrency; defaults to `1`.
|
||||
- `server.http.retention`: optional completed upload retention duration; defaults to `24h`.
|
||||
- `secrets.directory`: optional credential secrets directory.
|
||||
- `pipelines`: required non-empty list.
|
||||
|
||||
@@ -170,6 +265,9 @@ Source backend:
|
||||
- `force_path_style`: optional for `s3`; defaults to `true`. Set `false` only for services that require virtual-host addressing.
|
||||
- `credentials.access_key_id_env`: optional S3 credential environment variable name.
|
||||
- `credentials.secret_access_key_env`: optional S3 credential environment variable name.
|
||||
- `token_env`: required for `http_upload`; names the token environment variable or secret-file name.
|
||||
- `staging_path`: optional for `http_upload`; defaults below `server.http.staging_root` using the pipeline id.
|
||||
- `max_upload_size`: optional for `http_upload`; defaults to `server.http.max_upload_size`.
|
||||
|
||||
Destination:
|
||||
|
||||
@@ -187,6 +285,20 @@ Accepted backend names:
|
||||
- `local`: executable; requires `path`.
|
||||
- `ssh`: executable; requires `host` and `path`.
|
||||
- `s3`: executable; requires `endpoint` and `bucket`.
|
||||
- `http_upload`: source-only configuration; requires `token_env`.
|
||||
|
||||
## Size And Duration Values
|
||||
|
||||
Upload size fields use an integer plus one of the supported binary-size suffixes:
|
||||
|
||||
- `B`
|
||||
- `KB`
|
||||
- `MB`
|
||||
- `GB`
|
||||
|
||||
Suffix multipliers use powers of 1024. Size values must be greater than zero after defaults are applied.
|
||||
|
||||
HTTP retention uses Go-style duration strings such as `24h`, `90m`, or `168h`. Retention must be greater than zero after defaults are applied.
|
||||
|
||||
## SSH Backend
|
||||
|
||||
@@ -266,6 +378,14 @@ Defaults are applied after YAML decoding and before validation:
|
||||
- SSH `host_key_policy: accept-new`
|
||||
- S3 `region: us-east-1`
|
||||
- S3 `force_path_style: true`
|
||||
- `server.http.bind: 127.0.0.1:8080`
|
||||
- `server.http.staging_root: /var/spool/distributor`
|
||||
- `server.http.max_upload_size: 20MB`
|
||||
- `server.http.queue_size: 16`
|
||||
- `server.http.max_concurrency: 1`
|
||||
- `server.http.retention: 24h`
|
||||
- `source.staging_path: /var/spool/distributor/<pipeline id>` for `http_upload`
|
||||
- `source.max_upload_size: server.http.max_upload_size` for `http_upload`
|
||||
- `transform.markdown_to_html.mode: sidecar` when a Markdown-to-HTML transform block is present and mode is omitted
|
||||
- `publish.source: true`
|
||||
- `publish.html: false`
|
||||
@@ -294,6 +414,10 @@ S3 credentials may name environment variables:
|
||||
- `credentials.access_key_id_env`
|
||||
- `credentials.secret_access_key_env`
|
||||
|
||||
HTTP upload tokens name one environment variable or secret-file name:
|
||||
|
||||
- `source.token_env`
|
||||
|
||||
## Examples
|
||||
|
||||
Maintained examples live under [examples](../examples/):
|
||||
@@ -304,5 +428,6 @@ Maintained examples live under [examples](../examples/):
|
||||
- `local-index.yml`: runnable local `index.html` publication.
|
||||
- `fan-out.yml`: runnable local fan-out publication to source and HTML destinations.
|
||||
- `archive-and-latest.yml`: runnable local fan-out publication to an archive destination and a fixed latest destination.
|
||||
- `http-upload-local.yml`: local HTTP upload server example with a token environment variable reference.
|
||||
- `ssh-destination.yml`: environment-gated local-to-SSH publication example.
|
||||
- `s3-destination.yml`: environment-gated local-to-S3 publication example.
|
||||
|
||||
@@ -2,66 +2,232 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
`internal/app` owns top-level use cases for `run`, `validate`, and `inspect`. It wires configuration, storage backends, transforms, publish planning, execution, summaries, and notification handoff.
|
||||
`internal/app` owns the top-level application use cases. It coordinates
|
||||
configuration loading, secret resolution, backend construction, source bundle
|
||||
discovery, destination selection, publish planning, publish execution,
|
||||
notification handoff, run reporting, and in-memory run coordination.
|
||||
|
||||
## Inputs and outputs
|
||||
The package is the boundary between callers and lower-level domain packages. It
|
||||
does not own manifest validation rules, destination state comparison, storage
|
||||
path rules, output planning, transform rendering, or backend-specific behavior.
|
||||
|
||||
`Run` accepts a context, optional config path, dry-run flag, force flag, stdout writer, output format, and optional notifier. It loads YAML config, discovers source bundles for each configured pipeline, plans each destination independently, optionally executes publish plans, writes text or JSON output when stdout is supplied, and returns an aggregated error if any destination fails.
|
||||
## Use Cases
|
||||
|
||||
`Validate` and `Inspect` accept either a local path or one configured pipeline source. `Validate` discovers and validates bundles. `Inspect` writes bundle metadata and manifest file entries to stdout when provided.
|
||||
`Run` is the CLI-facing all-pipeline entrypoint. It accepts a context, optional
|
||||
config path, dry-run flag, force flag, stdout writer, output format, and
|
||||
optional notifier. It runs every configured pipeline, builds a `RunReport`, and
|
||||
projects the report to text or JSON when stdout is supplied.
|
||||
|
||||
## Run flow
|
||||
`RunPipeline` is the app-layer single-pipeline entrypoint. It accepts a context,
|
||||
config path, pipeline ID, dry-run flag, force flag, and optional notifier. It
|
||||
loads the same config as `Run`, narrows execution to exactly one configured
|
||||
pipeline, and returns a `RunReport` without writing command output.
|
||||
|
||||
The runner:
|
||||
`RunPipelineWithLocalSource` is the app-layer single-pipeline entrypoint for an
|
||||
already prepared local source bundle root. It accepts the same pipeline
|
||||
selection and execution options as `RunPipeline` plus a local source root path.
|
||||
It loads config, selects one configured pipeline, opens the supplied source
|
||||
root as a local backend, validates exactly that root bundle, and then uses the
|
||||
same destination fan-out path as normal runs.
|
||||
|
||||
`Validate` and `Inspect` accept either a local path or one configured pipeline
|
||||
source. They share source backend construction with run workflows and never open
|
||||
destination backends.
|
||||
|
||||
`Serve` is the CLI-facing HTTP upload server entrypoint. It loads config,
|
||||
loads the configured secrets directory, resolves upload bearer tokens for
|
||||
configured `http_upload` sources, creates an `UploadCoordinator`, binds
|
||||
`server.http.bind`, and serves the upload API until its context is cancelled.
|
||||
|
||||
## Run Reports
|
||||
|
||||
`RunReport` is the structured result model for run workflows. It includes
|
||||
dry-run state, pipeline summaries, action records, output metadata, summary
|
||||
counters, warnings, and destination-scoped output errors.
|
||||
|
||||
Text and JSON run output are projections of `RunReport`. JSON tags on report
|
||||
records match the CLI JSON output contract. Text output preserves the CLI
|
||||
summary shape while keeping output rendering outside the core planning and
|
||||
execution loop.
|
||||
|
||||
Destination-scoped failures produce a report plus an aggregated error. Fatal
|
||||
setup failures, such as config loading, source open, or source discovery
|
||||
failures, return before a complete run report is available.
|
||||
|
||||
## Run Flow
|
||||
|
||||
The app runner:
|
||||
|
||||
1. loads config from the supplied path or `config.DefaultConfigPath`;
|
||||
2. opens the configured source backend;
|
||||
3. discovers validated bundles from the source root;
|
||||
4. selects source bundles for each destination according to destination path mapping;
|
||||
5. opens each destination backend independently;
|
||||
6. builds publish plans for the selected bundle and destination combinations;
|
||||
7. prints plan lines or JSON action records and records summary counters;
|
||||
8. executes publish or replacement plans unless dry-run is enabled;
|
||||
9. invokes the notifier after successful publish or replacement actions.
|
||||
2. loads configured secret files into a config-owned environment resolver;
|
||||
3. builds the app-level backend factory and transform registry;
|
||||
4. opens each selected pipeline source backend;
|
||||
5. discovers validated source bundles from the source root;
|
||||
6. selects source bundles for each destination according to path mapping;
|
||||
7. opens destination backends independently;
|
||||
8. builds publish plans for selected bundle and destination combinations;
|
||||
9. records warnings, action records, output metadata, and summary counters;
|
||||
10. executes publish or replacement plans unless dry-run is enabled;
|
||||
11. invokes the notifier after successful publish or replacement actions;
|
||||
12. returns the structured report and any aggregated destination failures.
|
||||
|
||||
Destination failures are collected while later destinations continue to run. Source open and source discovery failures stop the run because there are no valid bundles to fan out.
|
||||
`RunPipeline` follows the same flow after selecting a single configured
|
||||
pipeline. It uses the same backend factory, secret loading, transform registry,
|
||||
warning generation, destination planning, publish execution, notification
|
||||
behavior, and failure aggregation as `Run`.
|
||||
|
||||
## Run implementation
|
||||
`RunPipelineWithLocalSource` follows the same flow after pipeline selection
|
||||
except for source opening and source discovery. It opens the supplied local
|
||||
source root directly, validates the root bundle before opening any destinations,
|
||||
and passes the resulting local source backend and bundle into the same
|
||||
destination planning and execution loop. Destination code receives the normal
|
||||
storage backend and bundle values and does not depend on how the source root was
|
||||
prepared.
|
||||
|
||||
`run.go` contains the public `Run` entrypoint and the main configuration orchestration path. Package-local run helpers are grouped by responsibility:
|
||||
## Upload Coordination
|
||||
|
||||
- `run_selection.go`: destination bundle selection, path mapping decisions, and fixed-path warnings;
|
||||
- `run_warnings.go`: secret and SSH warning data;
|
||||
- `run_output.go`: text plan lines, JSON action records, and output projections;
|
||||
- `run_summary.go`: summary counters and JSON summary records;
|
||||
- `run_failures.go`: destination failure aggregation and partial-result detection;
|
||||
`UploadCoordinator` owns in-memory coordination for asynchronous upload
|
||||
processing. It admits uploads for configured `http_upload` pipelines, generates
|
||||
run IDs, tracks status records, stages accepted archives through
|
||||
`internal/ingest`, and executes the selected pipeline through
|
||||
`RunPipelineWithLocalSource`.
|
||||
|
||||
Upload run IDs use:
|
||||
|
||||
```text
|
||||
<pipeline id>.<UTC timestamp>.<random suffix>
|
||||
```
|
||||
|
||||
The timestamp uses `YYYYMMDDThhmmssZ` UTC format and the suffix is filesystem
|
||||
safe.
|
||||
|
||||
The coordinator records these statuses:
|
||||
|
||||
- `accepted`
|
||||
- `queued`
|
||||
- `running`
|
||||
- `succeeded`
|
||||
- `failed`
|
||||
- `expired`
|
||||
|
||||
Admission is bounded by `server.http.queue_size`. Full queues are rejected
|
||||
before the upload body is staged. Execution is bounded by
|
||||
`server.http.max_concurrency`, and only one upload for a given pipeline may run
|
||||
at a time. Later uploads for the same pipeline remain queued until the active
|
||||
run finishes.
|
||||
|
||||
Completed records retain the final run report or error text until
|
||||
`server.http.retention` elapses. Expiration removes completed status records and
|
||||
their committed staged bundle directories. The coordinator is memory-only and
|
||||
does not persist queue state, status records, or run reports.
|
||||
|
||||
## HTTP Upload Server
|
||||
|
||||
The HTTP upload server is app-layer transport wiring around
|
||||
`UploadCoordinator`. It owns request authentication, route dispatch, HTTP status
|
||||
mapping, and JSON response projection. Bundle staging and publication remain in
|
||||
the coordinator and staged-source run path.
|
||||
|
||||
Server startup resolves every configured `http_upload` source `token_env`
|
||||
through the config-owned environment resolver after `secrets.directory` has
|
||||
been loaded. Startup fails when a token is missing, empty, or duplicates another
|
||||
upload pipeline token. Error messages identify environment variable names and
|
||||
pipeline ids, but not token values.
|
||||
|
||||
Routes:
|
||||
|
||||
- `GET /healthz`: returns `200` after config, secrets, tokens, coordinator, and route setup succeed.
|
||||
- `POST /upload`: accepts authenticated tar and tar.gz archives and returns an accepted run id.
|
||||
- `GET /runs/<run_id>`: returns the current in-memory upload status record or `404`.
|
||||
|
||||
The upload token maps to exactly one configured pipeline. Producers do not
|
||||
submit pipeline ids, and submitted `pipeline` or `pipeline_id` query values are
|
||||
rejected. Full queues are rejected before the request body is read. Oversized
|
||||
uploads, unsupported content types, invalid bearer tokens, full queues, and
|
||||
unknown status records are mapped to stable HTTP status codes without returning
|
||||
secret token values.
|
||||
|
||||
## Coordination
|
||||
|
||||
`PipelineRunCoordinator` wraps `RunPipeline` with in-memory admission control.
|
||||
It allows different pipeline IDs to run concurrently and rejects a second active
|
||||
run for the same pipeline ID.
|
||||
|
||||
Coordinator records contain a run ID, pipeline ID, status, timestamps, completed
|
||||
report, and error text when applicable. Active state is memory-only and is
|
||||
cleared after success, failure, unknown pipeline ID, or context cancellation.
|
||||
|
||||
The admission context is checked before a run is accepted. Once accepted, the
|
||||
run uses the coordinator lifetime context, so caller cancellation can stop
|
||||
waiting for admission without owning the actual run lifetime.
|
||||
|
||||
The coordinator does not queue duplicate runs, persist run records, or define
|
||||
transport endpoints.
|
||||
|
||||
## Errors
|
||||
|
||||
`Run` returns immediately for config loading errors, context cancellation before
|
||||
work starts, source open errors, and source discovery errors.
|
||||
|
||||
`RunPipeline` returns `PipelineNotFoundError` when the requested pipeline ID is
|
||||
not configured. Callers can detect that condition with `IsPipelineNotFound`.
|
||||
|
||||
`RunPipelineWithLocalSource` also returns `PipelineNotFoundError` for an unknown
|
||||
pipeline ID. It returns before destination opening when the supplied local
|
||||
source root is missing, cannot be opened, or does not validate as one complete
|
||||
source bundle.
|
||||
|
||||
Per-destination backend, planning, execution, and notification errors are
|
||||
aggregated into one run error after remaining destinations have been attempted.
|
||||
Destination diagnostics include pipeline ID, destination ID, backend, and
|
||||
bundle path.
|
||||
|
||||
`PipelineRunCoordinator` returns `DuplicatePipelineRunError` when the same
|
||||
pipeline already has an active run. Callers can detect that condition with
|
||||
`IsDuplicatePipelineRun`.
|
||||
|
||||
Stdout write errors are returned immediately because the caller's requested
|
||||
output stream can no longer be trusted.
|
||||
|
||||
## Package Layout
|
||||
|
||||
Run helpers are grouped by responsibility:
|
||||
|
||||
- `run.go`: `Run`, `RunPipeline`, and shared run orchestration.
|
||||
- `run_output.go`: `RunReport`, action/output records, and text/JSON report projection.
|
||||
- `run_summary.go`: summary counters.
|
||||
- `run_failures.go`: destination failure aggregation and partial-result detection.
|
||||
- `run_selection.go`: destination bundle selection, path mapping decisions, and fixed-path warnings.
|
||||
- `run_warnings.go`: secret and SSH warning records.
|
||||
- `run_notify.go`: notification event projection and action filtering.
|
||||
- `run_coordinator.go`: in-memory run admission, run IDs, status records, and duplicate-run errors.
|
||||
- `upload_coordinator.go`: in-memory upload admission, queueing, status tracking, staging handoff, and staged-source execution.
|
||||
- `upload_http.go`: HTTP upload authentication, routes, JSON response projection, and HTTP error mapping.
|
||||
- `serve.go`: config/secrets loading and HTTP server startup.
|
||||
- `backends.go`: app-level backend factory wiring.
|
||||
- `transforms.go`: app-level transform registry wiring.
|
||||
- `source_select.go`: configured-source selection shared by `validate` and `inspect`.
|
||||
|
||||
These helpers remain in `internal/app` because command output, warning collection, destination failure aggregation, notifier handoff, and backend construction are app-owned orchestration concerns.
|
||||
## Backend And Transform Wiring
|
||||
|
||||
## Backend and transform wiring
|
||||
The app-level backend factory registers local, SSH, and S3 backends for runtime
|
||||
execution. Source and destination backend config is converted through a shared
|
||||
app-local open spec before adapter construction.
|
||||
|
||||
The app-level backend factory registers local, SSH, and S3 backends for execution. Source and destination backend config is converted through a shared app-local open spec before adapter construction. S3 explicit credential references are resolved through the config environment resolver.
|
||||
Credential references are resolved through the config environment resolver.
|
||||
Production app code must not read backend credential environment variables
|
||||
directly.
|
||||
|
||||
The app-level transform registry registers Markdown-to-HTML using `internal/transform/markdown`. Lower-level publish code receives a resolver and does not import concrete transform implementations.
|
||||
The app-level transform registry registers Markdown-to-HTML through
|
||||
`internal/transform/markdown`. Lower-level publish code receives a resolver and
|
||||
does not import concrete transform implementations.
|
||||
|
||||
## Dry-run behavior
|
||||
## Dry-Run Behavior
|
||||
|
||||
Dry-run still loads config, opens backends, discovers bundles, inspects destinations, resolves transforms, and builds publish plans. It does not write destination outputs, write `.distributor.json`, delete managed outputs, perform forced prefix deletion, or notify.
|
||||
|
||||
## Failure behavior
|
||||
|
||||
`Run` returns immediately for config loading errors, context cancellation before work starts, source open errors, and source discovery errors. Per-destination backend, planning, execution, and notification errors are aggregated into one run error after remaining destinations have been attempted.
|
||||
|
||||
Run diagnostics include pipeline id, destination id, destination backend, and bundle path for destination-scoped failures. Source open and discovery failures include the source backend.
|
||||
|
||||
Stdout write errors are returned immediately because the caller's requested output stream can no longer be trusted.
|
||||
|
||||
## Boundaries
|
||||
|
||||
`internal/app` coordinates packages but does not own manifest validation rules, destination state comparison, storage path rules, output planning, transform rendering, or backend-specific filesystem behavior.
|
||||
|
||||
Configured-source `Validate` and `Inspect` share source backend construction with `Run` and do not open destinations.
|
||||
Dry-run loads config, opens backends, discovers bundles, inspects destinations,
|
||||
resolves transforms, and builds publish plans. It does not write destination
|
||||
outputs, write `.distributor.json`, delete managed outputs, perform forced
|
||||
prefix deletion, or invoke notifications.
|
||||
|
||||
## Tests
|
||||
|
||||
@@ -71,10 +237,18 @@ Before changing app orchestration, inspect tests under:
|
||||
- `internal/cli`
|
||||
- `internal/publish`
|
||||
|
||||
Use focused app tests for report structure, single-pipeline execution,
|
||||
coordinator admission, warning generation, notification behavior, and
|
||||
partial-result aggregation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- One source fans out to each destination independently.
|
||||
- Destination failures do not prevent later destinations from being planned.
|
||||
- Destination-scoped failures still produce a structured report plus an aggregated error.
|
||||
- Dry-run must not mutate destination storage or invoke notifications.
|
||||
- `RunPipeline` must use the same run path as `Run` after pipeline selection.
|
||||
- Duplicate in-flight runs are rejected only for the same pipeline ID.
|
||||
- Different pipeline IDs may run concurrently.
|
||||
- Concrete backend and transform registration stays at the app layer.
|
||||
- The default notifier is `notify.Noop`.
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
## Inputs and outputs
|
||||
|
||||
Input is a YAML file containing optional `secrets` and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
|
||||
Input is a YAML file containing optional `server`, optional `secrets`, and required `pipelines`. Output is a `Config` value with defaults applied and validation completed. Load failures include the config path and whether the failure occurred during file loading, YAML parsing, or validation.
|
||||
|
||||
## Loading flow
|
||||
|
||||
@@ -14,12 +14,21 @@ Input is a YAML file containing optional `secrets` and required `pipelines`. Out
|
||||
|
||||
Known-field checking rejects misspelled or unknown YAML keys before defaults and validation run.
|
||||
|
||||
`LoadFile` does not read secret files. `Run` loads the configured secrets directory after config validation and before backend construction.
|
||||
`LoadFile` does not read secret files. App entrypoints load the configured
|
||||
secrets directory after config validation and before credential-consuming work.
|
||||
|
||||
## Defaults
|
||||
|
||||
Defaults are applied in `ApplyDefaults`:
|
||||
|
||||
- HTTP server `bind` defaults to `127.0.0.1:8080`;
|
||||
- HTTP server `staging_root` defaults to `/var/spool/distributor`;
|
||||
- HTTP server `max_upload_size` defaults to `20MB`;
|
||||
- HTTP server `queue_size` defaults to `16`;
|
||||
- HTTP server `max_concurrency` defaults to `1`;
|
||||
- HTTP server `retention` defaults to `24h`;
|
||||
- `http_upload` source `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`;
|
||||
- `http_upload` source `max_upload_size` defaults to `server.http.max_upload_size`;
|
||||
- pipeline validation defaults `on_digest_mismatch` to `fail`;
|
||||
- SSH backend `port` defaults to `22`;
|
||||
- SSH backend `host_key_policy` defaults to `accept-new`;
|
||||
@@ -34,7 +43,11 @@ Defaults are applied in `ApplyDefaults`:
|
||||
|
||||
## Validation responsibilities
|
||||
|
||||
Validation requires at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions.
|
||||
Validation requires positive HTTP server limits and retention, at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions.
|
||||
|
||||
HTTP upload sources require `token_env`, a staging path after defaults, and a positive maximum upload size. Literal token fields are not part of the YAML schema. The `http_upload` backend is accepted only for sources and rejected for destinations.
|
||||
|
||||
Upload size values are parsed from strings with `B`, `KB`, `MB`, or `GB` suffixes using 1024 multipliers. Retention values are parsed with `time.ParseDuration`. Explicit zero values fail validation; omitted values receive defaults before validation.
|
||||
|
||||
Transfer validation accepts `replace` for `on_destination_newer` and `on_conflict`, but publish planning honors those destructive actions only when the current run explicitly requests force.
|
||||
|
||||
@@ -46,12 +59,14 @@ Destination links are optional. When a `links` block is present, `base_url` is r
|
||||
|
||||
## Executable support boundary
|
||||
|
||||
Config validation accepts `local`, `ssh`, and `s3` backend shapes. Runtime execution opens all three through `internal/app`.
|
||||
Config validation accepts `local`, `ssh`, `s3`, and source-only `http_upload` backend shapes. Runtime `run`, `validate`, and `inspect` workflows open `local`, `ssh`, and `s3` through `internal/app`. Runtime `serve` workflows execute `http_upload` sources through the app upload coordinator and HTTP server.
|
||||
|
||||
SSH config uses structured fields: `host`, optional `user`, optional `port`, `path`, optional `ssh_key_file`, optional `known_hosts`, and optional `host_key_policy`. `host_key_policy` accepts YAML booleans and strings and normalizes `true`/`strict`, `accept-new`, and `false`/`off`.
|
||||
|
||||
S3 config requires `endpoint` and `bucket`, normalizes optional `prefix`, defaults `region` to `us-east-1`, and defaults omitted `force_path_style` to `true` while preserving explicit `false`.
|
||||
|
||||
HTTP upload config is source-only. Config owns its YAML shape, defaulting, size and duration parsing, and validation. The config package does not authenticate requests, stage uploads, or execute HTTP upload sources. The app layer resolves `token_env` through the config-owned environment resolver before starting the HTTP server.
|
||||
|
||||
## Secrets and credential resolution
|
||||
|
||||
`secrets.directory` points to a directory of credential files. `LoadSecretEnvironment` reads regular files and symlinks to regular files, rejects invalid filenames, trims exactly one trailing LF or CRLF, and returns an `Environment` resolver plus conflict metadata.
|
||||
|
||||
43
docs/internal/ingest.md
Normal file
43
docs/internal/ingest.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Ingestion Internals
|
||||
|
||||
## Purpose
|
||||
|
||||
`internal/ingest` stages uploaded source bundle archives into local per-run directories. It does not authenticate requests, manage upload queues, publish destinations, or start an HTTP server.
|
||||
|
||||
## Archive staging
|
||||
|
||||
`StageArchive` accepts one upload body, content type, pipeline staging path, run id, and explicit size and file-count limits. It writes the request body to temporary storage while enforcing the configured upload size limit, extracts the archive into temporary local storage, validates the extracted source bundle, and then commits the validated bundle to:
|
||||
|
||||
```text
|
||||
<pipeline staging path>/<run id>
|
||||
```
|
||||
|
||||
The returned `StagedBundle.Root` is a local filesystem path to the validated source bundle root.
|
||||
|
||||
## Accepted archive formats
|
||||
|
||||
The package accepts only:
|
||||
|
||||
- `application/x-tar`
|
||||
- `application/gzip`
|
||||
- `application/x-gzip`
|
||||
|
||||
Gzip uploads must contain a tar archive.
|
||||
|
||||
## Extraction rules
|
||||
|
||||
Archive entry paths must be clean relative slash-separated paths. Extraction rejects absolute paths, path traversal, backslash paths, duplicate files, symlinks, hardlinks, devices, sockets, and other special entries.
|
||||
|
||||
The archive must contain exactly one root-level `manifest.json`. Nested manifests are rejected.
|
||||
|
||||
Regular files and directories are the only accepted tar entries. Regular file extraction enforces the explicit maximum extracted byte count and maximum file count supplied by the caller.
|
||||
|
||||
## Bundle validation
|
||||
|
||||
After extraction, the package loads and validates the staged bundle through `pkg/bundle`. Manifest parsing, source path validation, file existence checks, regular-file checks, file sizes, file SHA-256 digests, and bundle digest validation use the existing source bundle contract.
|
||||
|
||||
Validation happens before the staged bundle is committed to its final per-run path.
|
||||
|
||||
## Failure behavior
|
||||
|
||||
Failed staging removes temporary archive and extraction data created by the package. A failed call does not publish anything and does not leave a committed per-run bundle directory.
|
||||
@@ -68,6 +68,84 @@ Validate one configured source without opening destinations:
|
||||
go run ./cmd/distributor validate --config examples/local-publish.yml --pipeline example-source-bundle
|
||||
```
|
||||
|
||||
## HTTP Upload Workflow
|
||||
|
||||
`distributor serve` runs the HTTP upload API for pipelines whose source backend
|
||||
is `http_upload`. Each upload token maps to one configured pipeline, and each
|
||||
accepted archive is staged, validated, and published through the same
|
||||
destination fan-out path used by local source runs.
|
||||
|
||||
Minimal local HTTP upload configuration:
|
||||
|
||||
```yaml
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:8080
|
||||
staging_root: /var/spool/distributor
|
||||
max_upload_size: 20MB
|
||||
queue_size: 16
|
||||
max_concurrency: 1
|
||||
retention: 24h
|
||||
secrets:
|
||||
directory: /run/secrets/distributor
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /srv/reports/archive
|
||||
```
|
||||
|
||||
Create `/run/secrets/distributor/DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` or set the
|
||||
real process environment variable before starting the server. Distributor does
|
||||
not read literal upload tokens from YAML.
|
||||
|
||||
Start the maintained local example:
|
||||
|
||||
```sh
|
||||
DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN=<token> \
|
||||
go run ./cmd/distributor serve --config examples/http-upload-local.yml
|
||||
```
|
||||
|
||||
Submit a tar or tar.gz source bundle:
|
||||
|
||||
```sh
|
||||
curl -X POST http://127.0.0.1:8080/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/gzip" \
|
||||
--data-binary @bundle.tar.gz
|
||||
```
|
||||
|
||||
Successful admission returns a run id:
|
||||
|
||||
```json
|
||||
{"run_id":"reports.20260603T120000Z.abcdef12","status":"accepted"}
|
||||
```
|
||||
|
||||
Poll status until it reaches `succeeded` or `failed`:
|
||||
|
||||
```sh
|
||||
curl http://127.0.0.1:8080/runs/<run-id>
|
||||
```
|
||||
|
||||
The status record includes the completed run report on successful publication
|
||||
or error details on failure. Status is memory-only and expires after
|
||||
`server.http.retention`; completed staged bundle directories are removed on
|
||||
expiry. Restarting the process clears upload status and queue state.
|
||||
|
||||
Use `GET /healthz` for readiness after config and tokens load:
|
||||
|
||||
```sh
|
||||
curl http://127.0.0.1:8080/healthz
|
||||
```
|
||||
|
||||
The default bind address is private loopback. Put TLS, public routing,
|
||||
rate-limiting, and external access policy in a reverse proxy or deployment
|
||||
layer.
|
||||
|
||||
## Filesystem Layout
|
||||
|
||||
Source bundles are discovered beneath the configured source root. Each bundle is a directory containing `manifest.json`.
|
||||
@@ -322,7 +400,10 @@ secrets:
|
||||
directory: /run/secrets/distributor
|
||||
```
|
||||
|
||||
The directory is loaded during `run` and configured-source `validate` or `inspect` before any backend is opened. If the directory is missing, unreadable, or contains an invalid secret filename, the command fails before storage work starts.
|
||||
The directory is loaded during `run`, `serve`, and configured-source `validate`
|
||||
or `inspect` before credential-consuming work starts. If the directory is
|
||||
missing, unreadable, or contains an invalid secret filename, the command fails
|
||||
before storage work starts.
|
||||
|
||||
Real process environment values take precedence over files with the same name. If the values differ and stdout is enabled, `run` and configured-source diagnostics print a warning naming the ignored secret file variable without printing either value. The process environment is not changed.
|
||||
|
||||
|
||||
60
docs/roadmap/http.md
Normal file
60
docs/roadmap/http.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# Roadmap: HTTP Upload Extensions
|
||||
|
||||
## Purpose
|
||||
|
||||
The HTTP upload API is implemented. Current behavior is documented in:
|
||||
|
||||
- [CLI](../cli.md)
|
||||
- [Configuration](../config.md)
|
||||
- [Operations](../operations.md)
|
||||
- [Troubleshooting](../troubleshooting.md)
|
||||
- [Application internals](../internal/app.md)
|
||||
- [Ingestion internals](../internal/ingest.md)
|
||||
|
||||
This roadmap records HTTP upload extensions that are intentionally not part of
|
||||
the current implementation.
|
||||
|
||||
## Deferred Extensions
|
||||
|
||||
### Authentication
|
||||
|
||||
- URL-token authentication for constrained clients.
|
||||
- Additional token lifecycle tooling.
|
||||
- Mutual TLS or other in-app identity mechanisms.
|
||||
|
||||
### Archive Formats
|
||||
|
||||
- Zstandard-compressed tar archives.
|
||||
- Additional content negotiation rules for future archive formats.
|
||||
|
||||
### Status And Queue Durability
|
||||
|
||||
- Durable status persistence across process restarts.
|
||||
- Database-backed queueing.
|
||||
- Recovery semantics for queued or running uploads after a restart.
|
||||
|
||||
### Producer Coordination
|
||||
|
||||
- Producer-supplied idempotency keys.
|
||||
- Run retry endpoints.
|
||||
- Run cancellation endpoints.
|
||||
- Run listing endpoints.
|
||||
|
||||
### Deployment Surface
|
||||
|
||||
- In-app TLS.
|
||||
- Public exposure defaults.
|
||||
- Browser UI.
|
||||
|
||||
## Boundaries
|
||||
|
||||
Current HTTP upload behavior remains intentionally small:
|
||||
|
||||
- `http_upload` is source-only and is not a durable storage backend.
|
||||
- Upload status is memory-only.
|
||||
- Producers submit complete tar or gzip-compressed tar source bundles.
|
||||
- Producers authenticate with `Authorization: Bearer <token>`.
|
||||
- Public access policy, TLS termination, and rate limiting belong outside
|
||||
`distributor` unless a future roadmap explicitly changes that boundary.
|
||||
|
||||
Do not document deferred extensions as available outside `docs/roadmap/`.
|
||||
@@ -1,102 +1,34 @@
|
||||
# Roadmap
|
||||
# HTTP Upload Deferred Work
|
||||
|
||||
This directory contains only future, deferred, or aspirational work for
|
||||
`distributor`. Implemented behavior is documented in the current user,
|
||||
operator, internal, policy, integration, and example documentation:
|
||||
## Purpose
|
||||
|
||||
- `README.md`
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/internal/`
|
||||
- `docs/integrations/markdown.md`
|
||||
- `docs/policy/`
|
||||
- `examples/`
|
||||
HTTP upload behavior is implemented and documented in the current-behavior
|
||||
manuals:
|
||||
|
||||
`distributor` currently supports local, SSH/SFTP, and S3-compatible source and
|
||||
destination backends; producer bundle creation through `pkg/bundle` and
|
||||
`distributor manifest create`; configured source validation and inspection;
|
||||
Markdown sidecar and `index.html` publication; archive and fixed destination
|
||||
path mapping; destination link metadata; shared text/JSON CLI output; and
|
||||
managed destination replacement behavior.
|
||||
- [CLI](../cli.md)
|
||||
- [Configuration](../config.md)
|
||||
- [Operations](../operations.md)
|
||||
- [Troubleshooting](../troubleshooting.md)
|
||||
- [Application internals](../internal/app.md)
|
||||
- [Configuration internals](../internal/config.md)
|
||||
- [Ingestion internals](../internal/ingest.md)
|
||||
|
||||
## Future Work
|
||||
This file tracks only HTTP upload work that is not implemented.
|
||||
|
||||
These items are not implemented. They should not be documented as current
|
||||
behavior outside `docs/roadmap/` unless a future implementation adds them.
|
||||
## Deferred Work
|
||||
|
||||
### CLI And Status Output
|
||||
- URL-token authentication.
|
||||
- Zstandard-compressed archive support.
|
||||
- Durable status persistence across process restarts.
|
||||
- Database-backed queueing.
|
||||
- Producer-supplied idempotency keys.
|
||||
- Run listing, cancellation, and retry endpoints.
|
||||
- In-app TLS.
|
||||
- Public network exposure defaults.
|
||||
- Browser UI.
|
||||
|
||||
- Add a root-global output flag only if the command parser is later refactored
|
||||
around shared root options.
|
||||
- Add output formats beyond `text` and `json` only if a concrete consumer
|
||||
requires them.
|
||||
- Add a versioned JSON schema reference after the first JSON-capable release.
|
||||
- Add destination-state inspection behind an explicit flag such as
|
||||
`--with-destinations` if operators need fan-out status diagnostics from
|
||||
`inspect`.
|
||||
- Add additional status or inspection presentation for destination primary
|
||||
links beyond the current `run --format json` result model.
|
||||
## Documentation Rule
|
||||
|
||||
### Producer Workflows
|
||||
|
||||
- Add a no-write manifest creation mode, such as writing manifest JSON to
|
||||
stdout, if producer pipelines need to capture manifests directly.
|
||||
- Add broader producer workflow helpers, such as richer ignore rules or
|
||||
template scaffolding, if real producer use cases require them.
|
||||
- Add remote or storage-backed producer writers only if producer applications
|
||||
need to assemble bundles outside the local filesystem.
|
||||
|
||||
### Publication And Transform Behavior
|
||||
|
||||
- Add a separate collection or site-index transform if distributor needs
|
||||
multi-page aggregation.
|
||||
- Add richer transform metadata only if future state consumers need more than
|
||||
the transform name and output path.
|
||||
- Add custom HTML index output names only if fixed `index.html` is too limiting
|
||||
for real deployments.
|
||||
- Add richer fixed-destination source selection policies if deployments need
|
||||
something other than newest-by-`created`.
|
||||
- Add stricter handling for equal latest timestamps if timestamp ties become
|
||||
common in producer workflows.
|
||||
- Add higher-level status or approval workflows for fixed-root replacements if
|
||||
dry-run output is not enough operational protection.
|
||||
- Add richer link policies only if `auto`, `html`, and `source` prove
|
||||
insufficient.
|
||||
|
||||
### State And Compatibility
|
||||
|
||||
- Define a post-release destination state schema bump policy before introducing
|
||||
materially incompatible state changes.
|
||||
- Add warning-only digest mismatch handling only if an operator workflow needs
|
||||
publication to continue after validation failures.
|
||||
- Add compatibility parsing for legacy SSH URI config only if migration support
|
||||
is required.
|
||||
|
||||
### Backends, Security, And Deployment
|
||||
|
||||
- Add authentication mechanisms beyond the implemented SSH agent/key and S3
|
||||
credential paths only when a concrete backend workflow requires them.
|
||||
- Add broad recursive destination deletion outside managed bundle paths only if
|
||||
a future design can preserve the current safety boundary.
|
||||
- Add concurrent fan-out publishing only if runtime profiling shows it is
|
||||
needed.
|
||||
- Add streaming, resumable, or multipart S3 uploads only if object sizes make
|
||||
the current write path insufficient.
|
||||
- Add cloud-provider-specific IAM integration docs only when the repository
|
||||
includes tested provider-specific behavior.
|
||||
- Add repository-managed packaging, release, and deployment automation when the
|
||||
release process is ready to be standardized.
|
||||
|
||||
## Roadmap Maintenance
|
||||
|
||||
When adding future roadmap work:
|
||||
|
||||
- describe user-visible behavior and safety boundaries;
|
||||
- define which current docs must change after implementation;
|
||||
- keep examples secret-free and runnable or clearly environment-gated;
|
||||
- keep workflow labels out of production code, tests, config fields, and
|
||||
user-facing documentation;
|
||||
- run focused tests for the changed behavior and `go test ./...` for
|
||||
cross-package changes.
|
||||
Deferred behavior belongs under `docs/roadmap/` until implemented. Current
|
||||
behavior docs must describe only the active HTTP upload API, configuration,
|
||||
operation, troubleshooting, and internal package contracts.
|
||||
|
||||
@@ -26,7 +26,10 @@ Safe fix: compare the file to the reference in [configuration](config.md) and re
|
||||
|
||||
## `validate config ... backend ... is unsupported`
|
||||
|
||||
Likely cause: a source or destination uses a backend name other than `local`, `ssh`, or `s3`.
|
||||
Likely cause: a source or destination uses an unsupported backend name, or a
|
||||
command is trying to execute a backend that is valid only for another workflow.
|
||||
`run`, `validate`, and `inspect` execute `local`, `ssh`, and `s3` sources.
|
||||
`serve` executes `http_upload` sources.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -34,7 +37,127 @@ Diagnostic:
|
||||
rg -n "backend:" <config-path>
|
||||
```
|
||||
|
||||
Safe fix: use `backend: local`, `backend: ssh`, or `backend: s3` for executable workflows.
|
||||
Safe fix: use `backend: local`, `backend: ssh`, or `backend: s3` for normal
|
||||
source and destination workflows. Use `backend: http_upload` only for sources
|
||||
handled by `distributor serve`.
|
||||
|
||||
## `bind HTTP server ... address already in use`
|
||||
|
||||
Likely cause: another process is already listening on `server.http.bind`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ss -ltnp | rg '<port>'
|
||||
```
|
||||
|
||||
Safe fix: stop the conflicting process or configure a different
|
||||
`server.http.bind` value. The default bind address is `127.0.0.1:8080`.
|
||||
|
||||
## `upload token environment variable ... is not set`
|
||||
|
||||
Likely cause: a configured `http_upload` source references `token_env`, but the
|
||||
variable is absent from both the real process environment and
|
||||
`secrets.directory`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
env | cut -d= -f1 | rg '^<token-variable>$'
|
||||
ls -l <secrets-directory>/<token-variable>
|
||||
```
|
||||
|
||||
Safe fix: set the real environment variable or create a readable
|
||||
secrets-directory file with the same name. Do not place literal token values in
|
||||
YAML.
|
||||
|
||||
## `upload token environment variables ... resolve to the same value`
|
||||
|
||||
Likely cause: two configured `http_upload` pipelines resolve to the same bearer
|
||||
token value.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
rg -n 'token_env:' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: assign a distinct non-empty token value to each `http_upload`
|
||||
pipeline. Distributor does not print the duplicate token value.
|
||||
|
||||
## `POST /upload` returns `401`
|
||||
|
||||
Likely cause: the request is missing `Authorization: Bearer <token>` or the
|
||||
token does not match any configured `http_upload` pipeline.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
curl -i -X POST http://127.0.0.1:8080/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/x-tar" \
|
||||
--data-binary @bundle.tar
|
||||
```
|
||||
|
||||
Safe fix: use the token value resolved by the configured `token_env`. Do not
|
||||
include token values in logs or tickets.
|
||||
|
||||
## `POST /upload` returns `413`
|
||||
|
||||
Likely cause: the request body exceeds the selected pipeline's
|
||||
`source.max_upload_size` or the default `server.http.max_upload_size`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ls -lh bundle.tar bundle.tar.gz
|
||||
rg -n 'max_upload_size:' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: upload a smaller archive, remove unnecessary files from the source
|
||||
bundle, or raise the configured upload size limit.
|
||||
|
||||
## `POST /upload` returns `415`
|
||||
|
||||
Likely cause: the upload uses an unsupported content type. The server accepts
|
||||
uncompressed tar and gzip-compressed tar archives only.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
file bundle.tar.gz
|
||||
```
|
||||
|
||||
Safe fix: send `Content-Type: application/x-tar`, `application/gzip`, or
|
||||
`application/x-gzip`, matching the archive format.
|
||||
|
||||
## `POST /upload` returns `503`
|
||||
|
||||
Likely cause: the in-memory upload queue is full.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
rg -n 'queue_size|max_concurrency' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: retry after active uploads finish, or increase `server.http.queue_size`
|
||||
for the deployment.
|
||||
|
||||
## `GET /runs/<run_id>` returns `404`
|
||||
|
||||
Likely cause: the run id is wrong, the process restarted, or the completed
|
||||
status record expired after `server.http.retention`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
curl -i http://127.0.0.1:8080/runs/<run-id>
|
||||
rg -n 'retention:' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: use the exact `run_id` returned by `POST /upload`. If status retention
|
||||
is too short for operators, increase `server.http.retention`.
|
||||
|
||||
## `--format: format must be text or json`
|
||||
|
||||
|
||||
24
examples/http-upload-local.yml
Normal file
24
examples/http-upload-local.yml
Normal file
@@ -0,0 +1,24 @@
|
||||
# Local HTTP upload example.
|
||||
# Set DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN in the process environment or provide a
|
||||
# secrets-directory file with that name before running `distributor serve`.
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:8080
|
||||
staging_root: workspace/http-upload/staging
|
||||
max_upload_size: 20MB
|
||||
queue_size: 16
|
||||
max_concurrency: 1
|
||||
retention: 24h
|
||||
pipelines:
|
||||
- id: example-http-upload
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN
|
||||
destinations:
|
||||
- id: local-archive
|
||||
backend: local
|
||||
path: workspace/published/http-upload
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
|
||||
@@ -21,6 +21,23 @@ type RunOptions struct {
|
||||
Notifier notify.Notifier
|
||||
}
|
||||
|
||||
type RunPipelineOptions struct {
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
DryRun bool
|
||||
Force bool
|
||||
Notifier notify.Notifier
|
||||
}
|
||||
|
||||
type RunPipelineWithLocalSourceOptions struct {
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
SourceRoot string
|
||||
DryRun bool
|
||||
Force bool
|
||||
Notifier notify.Notifier
|
||||
}
|
||||
|
||||
func Run(ctx context.Context, options RunOptions) error {
|
||||
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||
return err
|
||||
@@ -40,90 +57,155 @@ func Run(ctx context.Context, options RunOptions) error {
|
||||
return runConfig(ctx, cfg, options)
|
||||
}
|
||||
|
||||
func RunPipeline(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RunReport{}, err
|
||||
}
|
||||
|
||||
configPath := options.ConfigPath
|
||||
if configPath == "" {
|
||||
configPath = config.DefaultConfigPath
|
||||
}
|
||||
cfg, err := config.LoadFile(configPath)
|
||||
if err != nil {
|
||||
return RunReport{}, err
|
||||
}
|
||||
return runPipelineConfig(ctx, cfg, options)
|
||||
}
|
||||
|
||||
func RunPipelineWithLocalSource(ctx context.Context, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RunReport{}, err
|
||||
}
|
||||
if options.SourceRoot == "" {
|
||||
return RunReport{}, fmt.Errorf("source root is required")
|
||||
}
|
||||
|
||||
configPath := options.ConfigPath
|
||||
if configPath == "" {
|
||||
configPath = config.DefaultConfigPath
|
||||
}
|
||||
cfg, err := config.LoadFile(configPath)
|
||||
if err != nil {
|
||||
return RunReport{}, err
|
||||
}
|
||||
return runPipelineConfigWithLocalSource(ctx, cfg, options)
|
||||
}
|
||||
|
||||
func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error {
|
||||
return runConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
|
||||
}
|
||||
|
||||
type backendFactoryProvider func(config.Environment) *backendFactory
|
||||
|
||||
func runPipelineConfig(ctx context.Context, cfg config.Config, options RunPipelineOptions) (RunReport, error) {
|
||||
return runPipelineConfigWithBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
|
||||
}
|
||||
|
||||
func runPipelineConfigWithLocalSource(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
return runPipelineConfigWithLocalSourceAndBackendFactory(ctx, cfg, options, newBackendFactoryWithEnvironment)
|
||||
}
|
||||
|
||||
func runPipelineConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineOptions, provider backendFactoryProvider) (RunReport, error) {
|
||||
pipeline, ok := findPipeline(cfg, options.PipelineID)
|
||||
if !ok {
|
||||
return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
||||
}
|
||||
return buildRunReportWithBackendFactory(ctx, config.Config{
|
||||
Server: cfg.Server,
|
||||
Secrets: cfg.Secrets,
|
||||
Pipelines: []config.Pipeline{pipeline},
|
||||
}, RunOptions{
|
||||
DryRun: options.DryRun,
|
||||
Force: options.Force,
|
||||
Notifier: options.Notifier,
|
||||
}, provider)
|
||||
}
|
||||
|
||||
func runPipelineConfigWithLocalSourceAndBackendFactory(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions, provider backendFactoryProvider) (RunReport, error) {
|
||||
pipeline, ok := findPipeline(cfg, options.PipelineID)
|
||||
if !ok {
|
||||
return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
||||
}
|
||||
return buildRunReport(ctx, config.Config{
|
||||
Server: cfg.Server,
|
||||
Secrets: cfg.Secrets,
|
||||
Pipelines: []config.Pipeline{pipeline},
|
||||
}, RunOptions{
|
||||
DryRun: options.DryRun,
|
||||
Force: options.Force,
|
||||
Notifier: options.Notifier,
|
||||
}, provider, &localSourceRoot{
|
||||
pipelineID: options.PipelineID,
|
||||
root: options.SourceRoot,
|
||||
})
|
||||
}
|
||||
|
||||
func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) error {
|
||||
report, err := buildRunReportWithBackendFactory(ctx, cfg, options, provider)
|
||||
if err != nil && !IsPartialResultError(err) {
|
||||
return err
|
||||
}
|
||||
if outputErr := WriteRunReport(options.Stdout, options.OutputFormat, report); outputErr != nil {
|
||||
return outputErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func buildRunReportWithBackendFactory(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider) (RunReport, error) {
|
||||
return buildRunReport(ctx, cfg, options, provider, nil)
|
||||
}
|
||||
|
||||
type localSourceRoot struct {
|
||||
pipelineID string
|
||||
root string
|
||||
}
|
||||
|
||||
func buildRunReport(ctx context.Context, cfg config.Config, options RunOptions, provider backendFactoryProvider, sourceRoot *localSourceRoot) (RunReport, error) {
|
||||
notifier := options.Notifier
|
||||
if notifier == nil {
|
||||
notifier = notify.Noop{}
|
||||
}
|
||||
jsonOutput := IsJSONOutput(options.OutputFormat)
|
||||
summary := runSummary{dryRun: options.DryRun}
|
||||
result := runResult{
|
||||
report := RunReport{
|
||||
DryRun: options.DryRun,
|
||||
Pipelines: []runPipelineResult{},
|
||||
Actions: []runActionResult{},
|
||||
Pipelines: []RunPipelineSummary{},
|
||||
Actions: []RunActionRecord{},
|
||||
}
|
||||
var warnings []OutputWarning
|
||||
var failures runFailures
|
||||
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return report, err
|
||||
}
|
||||
secretWarnings := secretConflictWarnings(secretLoad.Conflicts)
|
||||
if jsonOutput {
|
||||
warnings = append(warnings, secretWarnings...)
|
||||
} else if options.Stdout != nil {
|
||||
if err := writeWarnings(options.Stdout, secretWarnings); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
report.PreambleWarnings = append(report.PreambleWarnings, secretWarnings...)
|
||||
report.addWarnings(secretWarnings)
|
||||
backends := provider(secretLoad.Environment)
|
||||
backends.readOnlyKnownHosts = options.DryRun
|
||||
transforms := newTransformRegistry()
|
||||
if options.Stdout != nil && !jsonOutput {
|
||||
if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, pipeline := range cfg.Pipelines {
|
||||
pipelineWarnings := sshWarnings(pipeline)
|
||||
if jsonOutput {
|
||||
warnings = append(warnings, pipelineWarnings...)
|
||||
} else if options.Stdout != nil {
|
||||
if err := writeWarnings(options.Stdout, pipelineWarnings); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
|
||||
report.addWarnings(pipelineWarnings)
|
||||
sourceBackend, bundles, sourceBackendName, err := openPipelineSource(ctx, backends, pipeline, sourceRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||
return report, err
|
||||
}
|
||||
bundles, err := bundle.Discover(ctx, sourceBackend, "")
|
||||
if err != nil {
|
||||
closeBackend(sourceBackend)
|
||||
return fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||
}
|
||||
result.Pipelines = append(result.Pipelines, runPipelineResult{
|
||||
report.Pipelines = append(report.Pipelines, RunPipelineSummary{
|
||||
ID: pipeline.ID,
|
||||
SourceBackend: pipeline.Source.Backend,
|
||||
SourceBackend: sourceBackendName,
|
||||
BundleCount: len(bundles),
|
||||
Destinations: destinationIDs(pipeline.Destinations),
|
||||
Warnings: pipelineWarnings,
|
||||
})
|
||||
if options.Stdout != nil && !jsonOutput {
|
||||
if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
|
||||
closeBackend(sourceBackend)
|
||||
return err
|
||||
}
|
||||
}
|
||||
pipelineIndex := len(report.Pipelines) - 1
|
||||
for _, destination := range pipeline.Destinations {
|
||||
selections := selectDestinationBundles(destination, bundles)
|
||||
if isFixedPathDestination(destination) {
|
||||
summary.recordFixedPath()
|
||||
if options.DryRun {
|
||||
warning := fixedPathSelectionWarning(pipeline.ID, destination.ID, selections, len(bundles))
|
||||
if jsonOutput {
|
||||
warnings = append(warnings, warning)
|
||||
} else if options.Stdout != nil {
|
||||
if err := writeWarnings(options.Stdout, []OutputWarning{warning}); err != nil {
|
||||
closeBackend(sourceBackend)
|
||||
return err
|
||||
}
|
||||
}
|
||||
report.addWarning(warning)
|
||||
report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, warningEvent(warning))
|
||||
}
|
||||
}
|
||||
if len(selections) == 0 {
|
||||
@@ -134,11 +216,8 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
|
||||
for _, selection := range selections {
|
||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(selection.SourceBundle.RootRelativePath), err)
|
||||
summary.recordFailure()
|
||||
if jsonOutput {
|
||||
result.Actions = append(result.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, selection.SourceBundle.RootRelativePath, err))
|
||||
} else if options.Stdout != nil {
|
||||
writeErrorLine(options.Stdout, selection.SourceBundle.RootRelativePath, destination.ID, destination.Backend, err)
|
||||
}
|
||||
report.Actions = append(report.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, selection.SourceBundle.RootRelativePath, err))
|
||||
report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, actionEvent(len(report.Actions)-1))
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -189,22 +268,12 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
|
||||
plan.PathMapping = config.PathMappingFixed
|
||||
if options.DryRun && isDestructiveFixedPathAction(plan.Action) {
|
||||
warning := fixedPathReplacementWarning(plan)
|
||||
if jsonOutput {
|
||||
warnings = append(warnings, warning)
|
||||
} else if options.Stdout != nil {
|
||||
if err := writeWarnings(options.Stdout, []OutputWarning{warning}); err != nil {
|
||||
deferCloseDestination()
|
||||
closeBackend(sourceBackend)
|
||||
return err
|
||||
}
|
||||
}
|
||||
report.addWarning(warning)
|
||||
report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, warningEvent(warning))
|
||||
}
|
||||
}
|
||||
if jsonOutput {
|
||||
result.Actions = append(result.Actions, runActionFromPlan(destination.Backend, plan, err))
|
||||
} else if options.Stdout != nil {
|
||||
writePlanLine(options.Stdout, destination.Backend, plan, err)
|
||||
}
|
||||
report.Actions = append(report.Actions, runActionFromPlan(destination.Backend, plan, err))
|
||||
report.Pipelines[pipelineIndex].events = append(report.Pipelines[pipelineIndex].events, actionEvent(len(report.Actions)-1))
|
||||
if err != nil {
|
||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||
summary.recordFailure()
|
||||
@@ -230,20 +299,38 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
|
||||
}
|
||||
closeBackend(sourceBackend)
|
||||
}
|
||||
result.Summary = summary.Result()
|
||||
if jsonOutput {
|
||||
if err := WriteJSONEnvelope(options.Stdout, "run", len(failures.items) == 0, warnings, result, failures.outputErrors()); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if options.Stdout != nil {
|
||||
if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
report.Summary = summary.Result()
|
||||
report.OutputErrors = failures.outputErrors()
|
||||
if len(failures.items) > 0 {
|
||||
return failures
|
||||
return report, failures
|
||||
}
|
||||
return nil
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func openPipelineSource(ctx context.Context, backends *backendFactory, pipeline config.Pipeline, sourceRoot *localSourceRoot) (storage.Backend, []bundle.Bundle, string, error) {
|
||||
if sourceRoot != nil && sourceRoot.pipelineID == pipeline.ID {
|
||||
sourceBackend, err := backends.openLocalPath(ctx, sourceRoot.root)
|
||||
if err != nil {
|
||||
return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, config.BackendLocal, err)
|
||||
}
|
||||
sourceBundle, err := bundle.Validate(ctx, sourceBackend, "")
|
||||
if err != nil {
|
||||
closeBackend(sourceBackend)
|
||||
return nil, nil, config.BackendLocal, fmt.Errorf("pipeline %s source backend %s validate source bundle: %w", pipeline.ID, config.BackendLocal, err)
|
||||
}
|
||||
return sourceBackend, []bundle.Bundle{sourceBundle}, config.BackendLocal, nil
|
||||
}
|
||||
|
||||
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
|
||||
if err != nil {
|
||||
return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||
}
|
||||
bundles, err := bundle.Discover(ctx, sourceBackend, "")
|
||||
if err != nil {
|
||||
closeBackend(sourceBackend)
|
||||
return nil, nil, pipeline.Source.Backend, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||
}
|
||||
return sourceBackend, bundles, pipeline.Source.Backend, nil
|
||||
}
|
||||
|
||||
type closeableBackend interface {
|
||||
|
||||
131
internal/app/run_coordinator.go
Normal file
131
internal/app/run_coordinator.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type PipelineRunID string
|
||||
|
||||
type PipelineRunStatus string
|
||||
|
||||
const (
|
||||
PipelineRunRunning PipelineRunStatus = "running"
|
||||
PipelineRunSucceeded PipelineRunStatus = "succeeded"
|
||||
PipelineRunFailed PipelineRunStatus = "failed"
|
||||
)
|
||||
|
||||
type PipelineRunRecord struct {
|
||||
ID PipelineRunID `json:"id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
Status PipelineRunStatus `json:"status"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
Report RunReport `json:"report,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type DuplicatePipelineRunError struct {
|
||||
PipelineID string
|
||||
RunID PipelineRunID
|
||||
}
|
||||
|
||||
func (err DuplicatePipelineRunError) Error() string {
|
||||
if err.RunID == "" {
|
||||
return fmt.Sprintf("pipeline %q already has an active run", err.PipelineID)
|
||||
}
|
||||
return fmt.Sprintf("pipeline %q already has active run %s", err.PipelineID, err.RunID)
|
||||
}
|
||||
|
||||
func IsDuplicatePipelineRun(err error) bool {
|
||||
var duplicate DuplicatePipelineRunError
|
||||
return errors.As(err, &duplicate)
|
||||
}
|
||||
|
||||
type PipelineRunCoordinator struct {
|
||||
ctx context.Context
|
||||
run pipelineRunFunc
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
nextID uint64
|
||||
active map[string]PipelineRunRecord
|
||||
}
|
||||
|
||||
type pipelineRunFunc func(context.Context, RunPipelineOptions) (RunReport, error)
|
||||
|
||||
func NewPipelineRunCoordinator(ctx context.Context) *PipelineRunCoordinator {
|
||||
return newPipelineRunCoordinator(ctx, RunPipeline)
|
||||
}
|
||||
|
||||
func newPipelineRunCoordinator(ctx context.Context, run pipelineRunFunc) *PipelineRunCoordinator {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
return &PipelineRunCoordinator{
|
||||
ctx: ctx,
|
||||
run: run,
|
||||
now: time.Now,
|
||||
active: map[string]PipelineRunRecord{},
|
||||
}
|
||||
}
|
||||
|
||||
func (coordinator *PipelineRunCoordinator) RunPipeline(ctx context.Context, options RunPipelineOptions) (PipelineRunRecord, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return PipelineRunRecord{}, err
|
||||
}
|
||||
record, err := coordinator.admit(options.PipelineID)
|
||||
if err != nil {
|
||||
return PipelineRunRecord{}, err
|
||||
}
|
||||
defer coordinator.clear(options.PipelineID)
|
||||
|
||||
report, runErr := coordinator.run(coordinator.ctx, options)
|
||||
record.Report = report
|
||||
finishedAt := coordinator.now().UTC()
|
||||
record.FinishedAt = &finishedAt
|
||||
if runErr != nil {
|
||||
record.Status = PipelineRunFailed
|
||||
record.Error = runErr.Error()
|
||||
return record, runErr
|
||||
}
|
||||
record.Status = PipelineRunSucceeded
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func (coordinator *PipelineRunCoordinator) admit(pipelineID string) (PipelineRunRecord, error) {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
if active, ok := coordinator.active[pipelineID]; ok {
|
||||
return PipelineRunRecord{}, DuplicatePipelineRunError{
|
||||
PipelineID: pipelineID,
|
||||
RunID: active.ID,
|
||||
}
|
||||
}
|
||||
coordinator.nextID++
|
||||
record := PipelineRunRecord{
|
||||
ID: PipelineRunID(fmt.Sprintf("run-%016d", coordinator.nextID)),
|
||||
PipelineID: pipelineID,
|
||||
Status: PipelineRunRunning,
|
||||
StartedAt: coordinator.now().UTC(),
|
||||
}
|
||||
coordinator.active[pipelineID] = record
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func (coordinator *PipelineRunCoordinator) clear(pipelineID string) {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
delete(coordinator.active, pipelineID)
|
||||
}
|
||||
|
||||
func (coordinator *PipelineRunCoordinator) activeCount() int {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
return len(coordinator.active)
|
||||
}
|
||||
223
internal/app/run_coordinator_test.go
Normal file
223
internal/app/run_coordinator_test.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPipelineRunCoordinatorRejectsDuplicateActiveRun(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var startedOnce sync.Once
|
||||
coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
|
||||
startedOnce.Do(func() {
|
||||
close(started)
|
||||
})
|
||||
<-release
|
||||
return RunReport{}, nil
|
||||
})
|
||||
firstResult := make(chan runCoordinatorTestResult, 1)
|
||||
|
||||
go func() {
|
||||
record, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
|
||||
firstResult <- runCoordinatorTestResult{record: record, err: err}
|
||||
}()
|
||||
waitForSignal(t, started, "first run to start")
|
||||
|
||||
_, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
|
||||
if err == nil || !IsDuplicatePipelineRun(err) {
|
||||
t.Fatalf("RunPipeline() error = %v, want duplicate active run", err)
|
||||
}
|
||||
close(release)
|
||||
result := waitForRunResult(t, firstResult)
|
||||
if result.err != nil {
|
||||
t.Fatalf("first RunPipeline() error = %v", result.err)
|
||||
}
|
||||
if result.record.Status != PipelineRunSucceeded || result.record.ID == "" || result.record.FinishedAt == nil {
|
||||
t.Fatalf("first record = %#v, want succeeded completed record", result.record)
|
||||
}
|
||||
if got := coordinator.activeCount(); got != 0 {
|
||||
t.Fatalf("active count = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineRunCoordinatorAllowsDifferentActivePipelines(t *testing.T) {
|
||||
started := make(chan string, 2)
|
||||
release := make(chan struct{})
|
||||
coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
|
||||
started <- options.PipelineID
|
||||
<-release
|
||||
return RunReport{}, nil
|
||||
})
|
||||
firstResult := make(chan runCoordinatorTestResult, 1)
|
||||
secondResult := make(chan runCoordinatorTestResult, 1)
|
||||
|
||||
go func() {
|
||||
record, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports-one"})
|
||||
firstResult <- runCoordinatorTestResult{record: record, err: err}
|
||||
}()
|
||||
go func() {
|
||||
record, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports-two"})
|
||||
secondResult <- runCoordinatorTestResult{record: record, err: err}
|
||||
}()
|
||||
startedPipelines := map[string]bool{
|
||||
waitForPipelineID(t, started): true,
|
||||
waitForPipelineID(t, started): true,
|
||||
}
|
||||
if !startedPipelines["reports-one"] || !startedPipelines["reports-two"] {
|
||||
t.Fatalf("started pipelines = %#v, want both requested pipelines", startedPipelines)
|
||||
}
|
||||
if got := coordinator.activeCount(); got != 2 {
|
||||
t.Fatalf("active count = %d, want 2", got)
|
||||
}
|
||||
|
||||
close(release)
|
||||
first := waitForRunResult(t, firstResult)
|
||||
second := waitForRunResult(t, secondResult)
|
||||
if first.err != nil || second.err != nil {
|
||||
t.Fatalf("RunPipeline() errors = %v, %v; want nil", first.err, second.err)
|
||||
}
|
||||
if first.record.ID == second.record.ID {
|
||||
t.Fatalf("run IDs matched: %q", first.record.ID)
|
||||
}
|
||||
if got := coordinator.activeCount(); got != 0 {
|
||||
t.Fatalf("active count = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineRunCoordinatorClearsActiveRunAfterSuccess(t *testing.T) {
|
||||
coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
|
||||
return RunReport{DryRun: options.DryRun}, nil
|
||||
})
|
||||
|
||||
first, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports", DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatalf("first RunPipeline() error = %v", err)
|
||||
}
|
||||
second, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
|
||||
if err != nil {
|
||||
t.Fatalf("second RunPipeline() error = %v", err)
|
||||
}
|
||||
if first.Status != PipelineRunSucceeded || second.Status != PipelineRunSucceeded {
|
||||
t.Fatalf("statuses = %s, %s; want succeeded", first.Status, second.Status)
|
||||
}
|
||||
if !first.Report.DryRun {
|
||||
t.Fatalf("first report dry_run = false, want true")
|
||||
}
|
||||
if got := coordinator.activeCount(); got != 0 {
|
||||
t.Fatalf("active count = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineRunCoordinatorClearsActiveRunAfterFailure(t *testing.T) {
|
||||
runError := errors.New("run failed")
|
||||
attempt := 0
|
||||
coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
|
||||
attempt++
|
||||
if attempt == 1 {
|
||||
return RunReport{}, runError
|
||||
}
|
||||
return RunReport{}, nil
|
||||
})
|
||||
|
||||
first, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
|
||||
if !errors.Is(err, runError) {
|
||||
t.Fatalf("first RunPipeline() error = %v, want run failure", err)
|
||||
}
|
||||
if first.Status != PipelineRunFailed || first.Error != runError.Error() || first.FinishedAt == nil {
|
||||
t.Fatalf("first record = %#v, want failed completed record", first)
|
||||
}
|
||||
second, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
|
||||
if err != nil {
|
||||
t.Fatalf("second RunPipeline() error = %v", err)
|
||||
}
|
||||
if second.Status != PipelineRunSucceeded {
|
||||
t.Fatalf("second status = %s, want succeeded", second.Status)
|
||||
}
|
||||
if got := coordinator.activeCount(); got != 0 {
|
||||
t.Fatalf("active count = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineRunCoordinatorClearsActiveRunAfterCancellation(t *testing.T) {
|
||||
runContext, cancel := context.WithCancel(context.Background())
|
||||
coordinator := newPipelineRunCoordinator(runContext, func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
|
||||
return RunReport{}, ctx.Err()
|
||||
})
|
||||
cancel()
|
||||
|
||||
record, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("RunPipeline() error = %v, want context canceled", err)
|
||||
}
|
||||
if record.Status != PipelineRunFailed || record.Error != context.Canceled.Error() {
|
||||
t.Fatalf("record = %#v, want failed cancellation record", record)
|
||||
}
|
||||
_, err = coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "reports"})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("second RunPipeline() error = %v, want context canceled", err)
|
||||
}
|
||||
if IsDuplicatePipelineRun(err) {
|
||||
t.Fatalf("second RunPipeline() error = %v, want cancellation instead of duplicate", err)
|
||||
}
|
||||
if got := coordinator.activeCount(); got != 0 {
|
||||
t.Fatalf("active count = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineRunCoordinatorUnknownPipelineDoesNotRemainActive(t *testing.T) {
|
||||
coordinator := newPipelineRunCoordinator(context.Background(), func(ctx context.Context, options RunPipelineOptions) (RunReport, error) {
|
||||
return RunReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
||||
})
|
||||
|
||||
_, err := coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "missing"})
|
||||
if err == nil || !IsPipelineNotFound(err) {
|
||||
t.Fatalf("RunPipeline() error = %v, want pipeline not found", err)
|
||||
}
|
||||
_, err = coordinator.RunPipeline(context.Background(), RunPipelineOptions{PipelineID: "missing"})
|
||||
if err == nil || !IsPipelineNotFound(err) || IsDuplicatePipelineRun(err) {
|
||||
t.Fatalf("second RunPipeline() error = %v, want pipeline not found without duplicate", err)
|
||||
}
|
||||
if got := coordinator.activeCount(); got != 0 {
|
||||
t.Fatalf("active count = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
type runCoordinatorTestResult struct {
|
||||
record PipelineRunRecord
|
||||
err error
|
||||
}
|
||||
|
||||
func waitForSignal(t *testing.T, signal <-chan struct{}, name string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-signal:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("timed out waiting for %s", name)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForPipelineID(t *testing.T, pipelineIDs <-chan string) string {
|
||||
t.Helper()
|
||||
select {
|
||||
case pipelineID := <-pipelineIDs:
|
||||
return pipelineID
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("timed out waiting for pipeline start")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func waitForRunResult(t *testing.T, results <-chan runCoordinatorTestResult) runCoordinatorTestResult {
|
||||
t.Helper()
|
||||
select {
|
||||
case result := <-results:
|
||||
return result
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("timed out waiting for run result")
|
||||
return runCoordinatorTestResult{}
|
||||
}
|
||||
}
|
||||
@@ -10,61 +10,125 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error) {
|
||||
if w == nil {
|
||||
return
|
||||
func WriteRunReport(w io.Writer, format OutputFormat, report RunReport) error {
|
||||
if IsJSONOutput(format) {
|
||||
return WriteJSONEnvelope(w, "run", len(report.OutputErrors) == 0, report.Warnings, report, report.OutputErrors)
|
||||
}
|
||||
if planErr != nil {
|
||||
destinationID := plan.DestinationID
|
||||
return writeRunReportText(w, report)
|
||||
}
|
||||
|
||||
func writeRunReportText(w io.Writer, report RunReport) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
if err := writeWarnings(w, report.PreambleWarnings); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintf(w, "Configured pipelines: %d\n", len(report.Pipelines)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, pipeline := range report.Pipelines {
|
||||
if err := writeWarnings(w, pipeline.Warnings); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintf(w, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.SourceBackend, pipeline.BundleCount, destinationIDSummary(pipeline.Destinations)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, event := range pipeline.events {
|
||||
if event.warning != nil {
|
||||
if err := writeWarnings(w, []OutputWarning{*event.warning}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if event.actionIndex < 0 || event.actionIndex >= len(report.Actions) {
|
||||
continue
|
||||
}
|
||||
writeRunActionLine(w, report.Actions[event.actionIndex])
|
||||
}
|
||||
}
|
||||
_, err := fmt.Fprintln(w, report.Summary.Line())
|
||||
return err
|
||||
}
|
||||
|
||||
func writeRunActionLine(w io.Writer, action RunActionRecord) {
|
||||
if action.Action == "error" {
|
||||
destinationID := action.DestinationID
|
||||
if destinationID == "" {
|
||||
destinationID = "unknown"
|
||||
}
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, pathMappingSummary(plan), planErr.Error())
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", action.BundlePath, destinationID, action.Backend, pathMappingRecordSummary(action), action.Reason)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, pathMappingSummary(plan), plan.Action, outputSummary(plan.Outputs), plan.Reason)
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, outputRecordSummary(action.Outputs), action.Reason)
|
||||
}
|
||||
|
||||
func pathMappingSummary(plan publish.Plan) string {
|
||||
if plan.PathMapping != config.PathMappingFixed {
|
||||
func pathMappingRecordSummary(action RunActionRecord) string {
|
||||
if action.PathMapping != config.PathMappingFixed {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" path_mapping=fixed target=%s", storage.DisplayPath(plan.DestinationBundlePath))
|
||||
return fmt.Sprintf(" path_mapping=fixed target=%s", action.DestinationPath)
|
||||
}
|
||||
|
||||
func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, backend, err.Error())
|
||||
}
|
||||
|
||||
func outputSummary(outputs []publish.Output) string {
|
||||
func outputRecordSummary(outputs []RunOutputRecord) string {
|
||||
if len(outputs) == 0 {
|
||||
return "none"
|
||||
}
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.DestinationPath)
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return strings.Join(paths, ",")
|
||||
}
|
||||
|
||||
type runResult struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
Pipelines []runPipelineResult `json:"pipelines"`
|
||||
Actions []runActionResult `json:"actions"`
|
||||
Summary runSummaryResult `json:"summary"`
|
||||
func destinationIDSummary(ids []string) string {
|
||||
if len(ids) == 0 {
|
||||
return "none"
|
||||
}
|
||||
return strings.Join(ids, ",")
|
||||
}
|
||||
|
||||
type runPipelineResult struct {
|
||||
ID string `json:"id"`
|
||||
SourceBackend string `json:"source_backend"`
|
||||
BundleCount int `json:"bundle_count"`
|
||||
Destinations []string `json:"destinations"`
|
||||
type RunReport struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
Pipelines []RunPipelineSummary `json:"pipelines"`
|
||||
Actions []RunActionRecord `json:"actions"`
|
||||
Summary RunSummaryCounters `json:"summary"`
|
||||
Warnings []OutputWarning `json:"-"`
|
||||
OutputErrors []OutputError `json:"-"`
|
||||
PreambleWarnings []OutputWarning `json:"-"`
|
||||
}
|
||||
|
||||
type runActionResult struct {
|
||||
func (r *RunReport) addWarning(warning OutputWarning) {
|
||||
r.Warnings = append(r.Warnings, warning)
|
||||
}
|
||||
|
||||
func (r *RunReport) addWarnings(warnings []OutputWarning) {
|
||||
r.Warnings = append(r.Warnings, warnings...)
|
||||
}
|
||||
|
||||
type RunPipelineSummary struct {
|
||||
ID string `json:"id"`
|
||||
SourceBackend string `json:"source_backend"`
|
||||
BundleCount int `json:"bundle_count"`
|
||||
Destinations []string `json:"destinations"`
|
||||
Warnings []OutputWarning `json:"-"`
|
||||
events []runPipelineEvent
|
||||
}
|
||||
|
||||
type runPipelineEvent struct {
|
||||
warning *OutputWarning
|
||||
actionIndex int
|
||||
}
|
||||
|
||||
func warningEvent(warning OutputWarning) runPipelineEvent {
|
||||
return runPipelineEvent{warning: &warning, actionIndex: -1}
|
||||
}
|
||||
|
||||
func actionEvent(actionIndex int) runPipelineEvent {
|
||||
return runPipelineEvent{actionIndex: actionIndex}
|
||||
}
|
||||
|
||||
type RunActionRecord struct {
|
||||
PipelineID string `json:"pipeline_id,omitempty"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Backend string `json:"backend"`
|
||||
@@ -75,10 +139,10 @@ type runActionResult struct {
|
||||
Action string `json:"action"`
|
||||
PrimaryURL string `json:"primary_url,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Outputs []runOutputResult `json:"outputs"`
|
||||
Outputs []RunOutputRecord `json:"outputs"`
|
||||
}
|
||||
|
||||
type runOutputResult struct {
|
||||
type RunOutputRecord struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
SourcePath string `json:"source_path,omitempty"`
|
||||
@@ -88,13 +152,13 @@ type runOutputResult struct {
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActionResult {
|
||||
func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActionRecord {
|
||||
if planErr != nil {
|
||||
destinationID := plan.DestinationID
|
||||
if destinationID == "" {
|
||||
destinationID = "unknown"
|
||||
}
|
||||
return runActionResult{
|
||||
return RunActionRecord{
|
||||
PipelineID: plan.PipelineID,
|
||||
DestinationID: destinationID,
|
||||
Backend: backend,
|
||||
@@ -105,10 +169,10 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActi
|
||||
Action: "error",
|
||||
PrimaryURL: plan.PrimaryURL,
|
||||
Reason: planErr.Error(),
|
||||
Outputs: []runOutputResult{},
|
||||
Outputs: []RunOutputRecord{},
|
||||
}
|
||||
}
|
||||
return runActionResult{
|
||||
return RunActionRecord{
|
||||
PipelineID: plan.PipelineID,
|
||||
DestinationID: plan.DestinationID,
|
||||
Backend: backend,
|
||||
@@ -123,8 +187,8 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActi
|
||||
}
|
||||
}
|
||||
|
||||
func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) runActionResult {
|
||||
return runActionResult{
|
||||
func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) RunActionRecord {
|
||||
return RunActionRecord{
|
||||
PipelineID: pipelineID,
|
||||
DestinationID: destinationID,
|
||||
Backend: backend,
|
||||
@@ -132,15 +196,15 @@ func errorAction(pipelineID, destinationID, backend, bundlePath string, err erro
|
||||
DestinationPath: storage.DisplayPath(bundlePath),
|
||||
Action: "error",
|
||||
Reason: err.Error(),
|
||||
Outputs: []runOutputResult{},
|
||||
Outputs: []RunOutputRecord{},
|
||||
}
|
||||
}
|
||||
|
||||
func runOutputsFromPlan(outputs []publish.Output) []runOutputResult {
|
||||
results := make([]runOutputResult, 0, len(outputs))
|
||||
func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
|
||||
results := make([]RunOutputRecord, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
stateOutput := output.StateOutputFile()
|
||||
results = append(results, runOutputResult{
|
||||
results = append(results, RunOutputRecord{
|
||||
Path: stateOutput.Path,
|
||||
Kind: stateOutput.Kind,
|
||||
SourcePath: stateOutput.SourcePath,
|
||||
|
||||
@@ -3,7 +3,6 @@ package app
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
@@ -78,14 +77,3 @@ func destinationIDs(destinations []config.Destination) []string {
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func destinationSummary(destinations []config.Destination) string {
|
||||
if len(destinations) == 0 {
|
||||
return "none"
|
||||
}
|
||||
ids := make([]string, 0, len(destinations))
|
||||
for _, destination := range destinations {
|
||||
ids = append(ids, destination.ID)
|
||||
}
|
||||
return strings.Join(ids, ",")
|
||||
}
|
||||
|
||||
@@ -39,15 +39,7 @@ func (s *runSummary) recordFixedPath() {
|
||||
s.fixedPath++
|
||||
}
|
||||
|
||||
func (s runSummary) Line() string {
|
||||
status := "ok"
|
||||
if s.failures > 0 {
|
||||
status = "failed"
|
||||
}
|
||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun, s.fixedPath)
|
||||
}
|
||||
|
||||
type runSummaryResult struct {
|
||||
type RunSummaryCounters struct {
|
||||
Status string `json:"status"`
|
||||
Planned int `json:"planned"`
|
||||
PublishNew int `json:"publish_new"`
|
||||
@@ -59,12 +51,16 @@ type runSummaryResult struct {
|
||||
FixedPath int `json:"fixed_path"`
|
||||
}
|
||||
|
||||
func (s runSummary) Result() runSummaryResult {
|
||||
func (s RunSummaryCounters) Line() string {
|
||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.ReplaceOlder, s.ForceReplace, s.Skipped, s.Failed, s.DryRun, s.FixedPath)
|
||||
}
|
||||
|
||||
func (s runSummary) Result() RunSummaryCounters {
|
||||
status := "ok"
|
||||
if s.failures > 0 {
|
||||
status = "failed"
|
||||
}
|
||||
return runSummaryResult{
|
||||
return RunSummaryCounters{
|
||||
Status: status,
|
||||
Planned: s.planned,
|
||||
PublishNew: s.publishNew,
|
||||
|
||||
@@ -3,6 +3,7 @@ package app
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -220,6 +221,116 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineWithLocalSourcePublishesConfiguredDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
|
||||
report, err := RunPipelineWithLocalSource(context.Background(), RunPipelineWithLocalSourceOptions{
|
||||
ConfigPath: writeUploadPipelineConfig(t, destinationRoot),
|
||||
PipelineID: "reports",
|
||||
SourceRoot: sourceRoot,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunPipelineWithLocalSource() error = %v", err)
|
||||
}
|
||||
|
||||
if got, want := report.Summary.Status, "ok"; got != want {
|
||||
t.Fatalf("report status = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(report.Pipelines), 1; got != want {
|
||||
t.Fatalf("pipeline count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := report.Pipelines[0].SourceBackend, config.BackendLocal; got != want {
|
||||
t.Fatalf("source backend = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := report.Pipelines[0].BundleCount, 1; got != want {
|
||||
t.Fatalf("bundle count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(report.Actions), 1; got != want {
|
||||
t.Fatalf("action count = %d, want %d", got, want)
|
||||
}
|
||||
if report.Actions[0].PipelineID != "reports" || report.Actions[0].DestinationID != "archive" {
|
||||
t.Fatalf("action = %#v, want reports/archive action", report.Actions[0])
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||
}
|
||||
|
||||
func TestRunPipelineWithLocalSourceValidatesBeforeDestinationWrites(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeJSONManifest(t, sourceRoot, testutil.ValidManifest(testutil.BundleOptions{}))
|
||||
|
||||
_, err := RunPipelineWithLocalSource(context.Background(), RunPipelineWithLocalSourceOptions{
|
||||
ConfigPath: writeUploadPipelineConfig(t, destinationRoot),
|
||||
PipelineID: "reports",
|
||||
SourceRoot: sourceRoot,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("RunPipelineWithLocalSource() error = nil, want validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "validate source bundle") {
|
||||
t.Fatalf("RunPipelineWithLocalSource() error = %v, want source validation context", err)
|
||||
}
|
||||
entries, readErr := os.ReadDir(destinationRoot)
|
||||
if readErr != nil {
|
||||
t.Fatalf("ReadDir() error = %v", readErr)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("destination entries = %d, want no writes", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineWithLocalSourcePublishesToRegisteredDestinationBackends(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
s3Destination := fake.New()
|
||||
sshDestination := fake.New()
|
||||
cfg := config.Config{
|
||||
Pipelines: []config.Pipeline{{
|
||||
ID: "reports",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{TokenEnv: "UPLOAD_TOKEN"},
|
||||
},
|
||||
Destinations: []config.Destination{
|
||||
{
|
||||
ID: "object-archive",
|
||||
Backend: config.BackendS3,
|
||||
Endpoint: "http://s3.test",
|
||||
Bucket: "destination-bucket",
|
||||
},
|
||||
{
|
||||
ID: "ssh-archive",
|
||||
Backend: config.BackendSSH,
|
||||
Host: "ssh.test",
|
||||
Path: "/destination",
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
|
||||
"s3:destination-bucket": s3Destination,
|
||||
"ssh:/destination": sshDestination,
|
||||
})
|
||||
|
||||
report, err := runPipelineConfigWithLocalSourceAndBackendFactory(context.Background(), cfg, RunPipelineWithLocalSourceOptions{
|
||||
PipelineID: "reports",
|
||||
SourceRoot: sourceRoot,
|
||||
}, provider)
|
||||
if err != nil {
|
||||
t.Fatalf("runPipelineConfigWithLocalSourceAndBackendFactory() error = %v", err)
|
||||
}
|
||||
|
||||
if got, want := len(report.Actions), 2; got != want {
|
||||
t.Fatalf("action count = %d, want %d", got, want)
|
||||
}
|
||||
testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, sshDestination, "summary.txt", "Summary\n")
|
||||
}
|
||||
|
||||
func TestRunExplicitPreserveRelativePathMappingMatchesDefault(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -718,6 +829,176 @@ func TestRunJSONIncludesGeneratedOutputMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRunReportIncludesStructuredDryRunResults(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
configPath := testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingFixed, "https://reports.example.com/latest", config.LinkPrimaryAuto, false, true, config.TransformModeIndex)
|
||||
cfg, err := config.LoadFile(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
|
||||
report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment)
|
||||
if err != nil {
|
||||
t.Fatalf("buildRunReportWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.DryRun || report.Summary.Status != "ok" || !report.Summary.DryRun {
|
||||
t.Fatalf("report dry-run/status = dry_run:%t summary:%#v, want ok dry-run", report.DryRun, report.Summary)
|
||||
}
|
||||
if got, want := len(report.Pipelines), 1; got != want {
|
||||
t.Fatalf("pipeline count = %d, want %d", got, want)
|
||||
}
|
||||
pipeline := report.Pipelines[0]
|
||||
if pipeline.ID != "reports" || pipeline.SourceBackend != config.BackendLocal || pipeline.BundleCount != 1 || strings.Join(pipeline.Destinations, ",") != "archive" {
|
||||
t.Fatalf("pipeline summary = %#v, want reports/local bundle summary", pipeline)
|
||||
}
|
||||
if got, want := len(report.Warnings), 1; got != want {
|
||||
t.Fatalf("warning count = %d, want %d", got, want)
|
||||
}
|
||||
if !strings.Contains(report.Warnings[0].Message, "path_mapping=fixed candidates=1 selected_bundle=.") {
|
||||
t.Fatalf("warning = %#v, want fixed path selection", report.Warnings[0])
|
||||
}
|
||||
if got, want := len(report.Actions), 1; got != want {
|
||||
t.Fatalf("action count = %d, want %d", got, want)
|
||||
}
|
||||
action := report.Actions[0]
|
||||
if action.PipelineID != "reports" || action.DestinationID != "archive" || action.Action != "publish_new" || action.PrimaryURL != "https://reports.example.com/latest/" {
|
||||
t.Fatalf("action = %#v, want publish_new with primary URL", action)
|
||||
}
|
||||
if action.PathMapping != config.PathMappingFixed || action.DestinationPath != "." {
|
||||
t.Fatalf("action path mapping = %q destination path = %q, want fixed root", action.PathMapping, action.DestinationPath)
|
||||
}
|
||||
if got, want := len(action.Outputs), 1; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
output := action.Outputs[0]
|
||||
if output.Path != "index.html" || output.Kind != state.OutputKindGenerated || output.SourcePath != "report.md" || output.Transform != "markdown_to_html" || output.URL != "https://reports.example.com/latest/" {
|
||||
t.Fatalf("output = %#v, want generated index metadata", output)
|
||||
}
|
||||
if report.Summary.Planned != 1 || report.Summary.PublishNew != 1 || report.Summary.FixedPath != 1 || report.Summary.Failed != 0 {
|
||||
t.Fatalf("summary = %#v, want publish_new fixed path counters", report.Summary)
|
||||
}
|
||||
if len(report.OutputErrors) != 0 {
|
||||
t.Fatalf("output errors = %#v, want none", report.OutputErrors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
}
|
||||
cfg, err := config.LoadFile(writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination))
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
|
||||
report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{}, newBackendFactoryWithEnvironment)
|
||||
if err == nil || !IsPartialResultError(err) {
|
||||
t.Fatalf("buildRunReportWithBackendFactory() error = %v, want partial result error", err)
|
||||
}
|
||||
if report.Summary.Status != "failed" || report.Summary.Planned != 1 || report.Summary.PublishNew != 1 || report.Summary.Failed != 1 {
|
||||
t.Fatalf("summary = %#v, want one planned publish and one failure", report.Summary)
|
||||
}
|
||||
if got, want := len(report.Actions), 2; got != want {
|
||||
t.Fatalf("action count = %d, want %d", got, want)
|
||||
}
|
||||
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "error" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
|
||||
t.Fatalf("first action = %#v, want archive-one error", report.Actions[0])
|
||||
}
|
||||
if report.Actions[1].DestinationID != "archive-two" || report.Actions[1].Action != "publish_new" {
|
||||
t.Fatalf("second action = %#v, want archive-two publish_new", report.Actions[1])
|
||||
}
|
||||
if got, want := len(report.OutputErrors), 1; got != want {
|
||||
t.Fatalf("output error count = %d, want %d", got, want)
|
||||
}
|
||||
outputError := report.OutputErrors[0]
|
||||
if outputError.PipelineID != "reports" || outputError.DestinationID != "archive-one" || outputError.Backend != config.BackendLocal || outputError.BundlePath != "." || !strings.Contains(outputError.Message, "fail_unmanaged") {
|
||||
t.Fatalf("output error = %#v, want archive-one unmanaged failure", outputError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineRunsOnlyRequestedPipeline(t *testing.T) {
|
||||
firstSource := t.TempDir()
|
||||
secondSource := t.TempDir()
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
writeSourceBundle(t, firstSource, "", testBundleOptions{ID: "reports.one"})
|
||||
writeSourceBundle(t, secondSource, "", testBundleOptions{ID: "reports.two"})
|
||||
configPath := writeTwoPipelineConfig(t, firstSource, firstDestination, secondSource, secondDestination)
|
||||
notifier := &recordingNotifier{}
|
||||
|
||||
report, err := RunPipeline(context.Background(), RunPipelineOptions{
|
||||
ConfigPath: configPath,
|
||||
PipelineID: "reports-one",
|
||||
Notifier: notifier,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("RunPipeline() error = %v", err)
|
||||
}
|
||||
if got, want := len(report.Pipelines), 1; got != want {
|
||||
t.Fatalf("pipeline count = %d, want %d", got, want)
|
||||
}
|
||||
if report.Pipelines[0].ID != "reports-one" {
|
||||
t.Fatalf("pipeline id = %q, want reports-one", report.Pipelines[0].ID)
|
||||
}
|
||||
if got, want := len(report.Actions), 1; got != want {
|
||||
t.Fatalf("action count = %d, want %d", got, want)
|
||||
}
|
||||
if report.Actions[0].PipelineID != "reports-one" || report.Actions[0].Action != "publish_new" {
|
||||
t.Fatalf("action = %#v, want reports-one publish_new", report.Actions[0])
|
||||
}
|
||||
if got, want := len(notifier.events), 1; got != want {
|
||||
t.Fatalf("notification count = %d, want %d", got, want)
|
||||
}
|
||||
if notifier.events[0].PipelineID != "reports-one" {
|
||||
t.Fatalf("notification pipeline = %q, want reports-one", notifier.events[0].PipelineID)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(firstDestination, "report.md"), "# Report\nSunny.\n")
|
||||
if entries, err := os.ReadDir(secondDestination); err != nil || len(entries) != 0 {
|
||||
t.Fatalf("second destination entries = %v err=%v, want empty", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineUnknownIDReturnsNotFound(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
|
||||
_, err := RunPipeline(context.Background(), RunPipelineOptions{
|
||||
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||
PipelineID: "missing",
|
||||
})
|
||||
if err == nil || !IsPipelineNotFound(err) {
|
||||
t.Fatalf("RunPipeline() error = %v, want pipeline not found", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), `pipeline "missing" not found`) {
|
||||
t.Fatalf("RunPipeline() error = %v, want pipeline id in message", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStillRunsAllConfiguredPipelines(t *testing.T) {
|
||||
firstSource := t.TempDir()
|
||||
secondSource := t.TempDir()
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
writeSourceBundle(t, firstSource, "", testBundleOptions{ID: "reports.one"})
|
||||
writeSourceBundle(t, secondSource, "", testBundleOptions{ID: "reports.two"})
|
||||
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: writeTwoPipelineConfig(t, firstSource, firstDestination, secondSource, secondDestination),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(firstDestination, "report.md"), "# Report\nSunny.\n")
|
||||
testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
|
||||
}
|
||||
|
||||
func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -1346,6 +1627,44 @@ func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestina
|
||||
return testutil.WriteFanoutLocalConfig(t, sourceRoot, firstDestination, secondDestination)
|
||||
}
|
||||
|
||||
func writeUploadPipelineConfig(t *testing.T, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: UPLOAD_TOKEN
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
`)
|
||||
}
|
||||
|
||||
func writeTwoPipelineConfig(t *testing.T, firstSource, firstDestination, secondSource, secondDestination string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports-one
|
||||
source:
|
||||
backend: local
|
||||
path: `+firstSource+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+firstDestination+`
|
||||
- id: reports-two
|
||||
source:
|
||||
backend: local
|
||||
path: `+secondSource+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+secondDestination+`
|
||||
`)
|
||||
}
|
||||
|
||||
func writeConfigFile(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
@@ -1360,6 +1679,20 @@ func writeDestinationState(t *testing.T, root, relative string, manifest bundle.
|
||||
testutil.WriteDestinationState(t, root, relative, manifest, testutil.DestinationStateOptions{})
|
||||
}
|
||||
|
||||
func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
t.Fatalf("mkdir manifest root: %v", err)
|
||||
}
|
||||
data, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal manifest: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, bundle.ManifestName), append(data, '\n'), 0o600); err != nil {
|
||||
t.Fatalf("write manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readStateFile(t *testing.T, path string) state.DistributorState {
|
||||
t.Helper()
|
||||
return testutil.ReadDestinationState(t, path)
|
||||
|
||||
62
internal/app/serve.go
Normal file
62
internal/app/serve.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
type ServeOptions struct {
|
||||
ConfigPath string
|
||||
}
|
||||
|
||||
func Serve(ctx context.Context, options ServeOptions) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configPath := options.ConfigPath
|
||||
if configPath == "" {
|
||||
configPath = config.DefaultConfigPath
|
||||
}
|
||||
cfg, err := config.LoadFile(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
handler, err := newUploadHTTPHandler(ctx, cfg, secretLoad.Environment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
listener, err := net.Listen("tcp", cfg.Server.HTTP.Bind)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bind HTTP server %q: %w", cfg.Server.HTTP.Bind, err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
server := &http.Server{Handler: handler}
|
||||
shutdownDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(shutdownDone)
|
||||
<-ctx.Done()
|
||||
_ = server.Shutdown(context.Background())
|
||||
}()
|
||||
|
||||
err = server.Serve(listener)
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
<-shutdownDone
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
@@ -9,6 +10,19 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
type PipelineNotFoundError struct {
|
||||
ID string
|
||||
}
|
||||
|
||||
func (e PipelineNotFoundError) Error() string {
|
||||
return fmt.Sprintf("pipeline %q not found", e.ID)
|
||||
}
|
||||
|
||||
func IsPipelineNotFound(err error) bool {
|
||||
var notFound PipelineNotFoundError
|
||||
return errors.As(err, ¬Found)
|
||||
}
|
||||
|
||||
type sourceCommandOptions struct {
|
||||
CommandName string
|
||||
Path string
|
||||
@@ -70,7 +84,7 @@ func selectSourceBundlesFromConfig(ctx context.Context, cfg config.Config, optio
|
||||
}
|
||||
pipeline, ok := findPipeline(cfg, options.PipelineID)
|
||||
if !ok {
|
||||
return sourceSelection{}, fmt.Errorf("pipeline %q not found", options.PipelineID)
|
||||
return sourceSelection{}, PipelineNotFoundError{ID: options.PipelineID}
|
||||
}
|
||||
backends := provider(secretLoad.Environment)
|
||||
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
|
||||
|
||||
389
internal/app/upload_coordinator.go
Normal file
389
internal/app/upload_coordinator.go
Normal file
@@ -0,0 +1,389 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
|
||||
)
|
||||
|
||||
const DefaultUploadMaxFileCount = 4096
|
||||
|
||||
type UploadRunID string
|
||||
|
||||
type UploadStatus string
|
||||
|
||||
const (
|
||||
UploadStatusAccepted UploadStatus = "accepted"
|
||||
UploadStatusQueued UploadStatus = "queued"
|
||||
UploadStatusRunning UploadStatus = "running"
|
||||
UploadStatusSucceeded UploadStatus = "succeeded"
|
||||
UploadStatusFailed UploadStatus = "failed"
|
||||
UploadStatusExpired UploadStatus = "expired"
|
||||
)
|
||||
|
||||
type UploadRunRecord struct {
|
||||
ID UploadRunID `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
Status UploadStatus `json:"status"`
|
||||
AcceptedAt time.Time `json:"accepted_at"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
Report *RunReport `json:"report,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
StagedRoot string `json:"-"`
|
||||
}
|
||||
|
||||
type UploadRequest struct {
|
||||
PipelineID string
|
||||
ContentType string
|
||||
Body io.Reader
|
||||
DryRun bool
|
||||
Force bool
|
||||
MaxFileCount int
|
||||
}
|
||||
|
||||
type UploadQueueFullError struct {
|
||||
QueueSize int
|
||||
}
|
||||
|
||||
func (err UploadQueueFullError) Error() string {
|
||||
return fmt.Sprintf("upload queue is full with capacity %d", err.QueueSize)
|
||||
}
|
||||
|
||||
func IsUploadQueueFull(err error) bool {
|
||||
var full UploadQueueFullError
|
||||
return errors.As(err, &full)
|
||||
}
|
||||
|
||||
type UploadCoordinator struct {
|
||||
ctx context.Context
|
||||
cfg config.Config
|
||||
stage uploadStageFunc
|
||||
run uploadRunFunc
|
||||
now func() time.Time
|
||||
randomSuffix func() (string, error)
|
||||
retention time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
signal chan struct{}
|
||||
queueSize int
|
||||
maxConcurrency int
|
||||
runningCount int
|
||||
activePipeline map[string]bool
|
||||
pending []*uploadJob
|
||||
records map[UploadRunID]UploadRunRecord
|
||||
}
|
||||
|
||||
type uploadStageFunc func(context.Context, ingest.StageOptions) (ingest.StagedBundle, error)
|
||||
|
||||
type uploadRunFunc func(context.Context, config.Config, RunPipelineWithLocalSourceOptions) (RunReport, error)
|
||||
|
||||
type uploadJob struct {
|
||||
recordID UploadRunID
|
||||
request UploadRequest
|
||||
pipeline config.Pipeline
|
||||
}
|
||||
|
||||
type uploadCoordinatorHooks struct {
|
||||
stage uploadStageFunc
|
||||
run uploadRunFunc
|
||||
now func() time.Time
|
||||
randomSuffix func() (string, error)
|
||||
}
|
||||
|
||||
func NewUploadCoordinator(ctx context.Context, cfg config.Config) *UploadCoordinator {
|
||||
return newUploadCoordinator(ctx, cfg, uploadCoordinatorHooks{})
|
||||
}
|
||||
|
||||
func newUploadCoordinator(ctx context.Context, cfg config.Config, hooks uploadCoordinatorHooks) *UploadCoordinator {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
stage := hooks.stage
|
||||
if stage == nil {
|
||||
stage = ingest.StageArchive
|
||||
}
|
||||
run := hooks.run
|
||||
if run == nil {
|
||||
run = runPipelineConfigWithLocalSource
|
||||
}
|
||||
now := hooks.now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
randomSuffix := hooks.randomSuffix
|
||||
if randomSuffix == nil {
|
||||
randomSuffix = randomRunIDSuffix
|
||||
}
|
||||
coordinator := &UploadCoordinator{
|
||||
ctx: ctx,
|
||||
cfg: cfg,
|
||||
stage: stage,
|
||||
run: run,
|
||||
now: now,
|
||||
randomSuffix: randomSuffix,
|
||||
retention: cfg.Server.HTTP.Retention.AsDuration(),
|
||||
signal: make(chan struct{}, 1),
|
||||
queueSize: cfg.Server.HTTP.QueueSize,
|
||||
maxConcurrency: cfg.Server.HTTP.MaxConcurrency,
|
||||
activePipeline: map[string]bool{},
|
||||
records: map[UploadRunID]UploadRunRecord{},
|
||||
}
|
||||
go coordinator.dispatchLoop()
|
||||
return coordinator
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return UploadRunRecord{}, err
|
||||
}
|
||||
if request.Body == nil {
|
||||
return UploadRunRecord{}, fmt.Errorf("upload body is required")
|
||||
}
|
||||
pipeline, ok := findPipeline(coordinator.cfg, request.PipelineID)
|
||||
if !ok {
|
||||
return UploadRunRecord{}, PipelineNotFoundError{ID: request.PipelineID}
|
||||
}
|
||||
if pipeline.Source.Backend != config.BackendHTTPUpload {
|
||||
return UploadRunRecord{}, fmt.Errorf("pipeline %s source backend %s is not configured for uploads", pipeline.ID, pipeline.Source.Backend)
|
||||
}
|
||||
runID, err := coordinator.newRunID(pipeline.ID)
|
||||
if err != nil {
|
||||
return UploadRunRecord{}, err
|
||||
}
|
||||
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
coordinator.expireLocked(coordinator.now().UTC())
|
||||
if len(coordinator.pending) >= coordinator.queueSize {
|
||||
return UploadRunRecord{}, UploadQueueFullError{QueueSize: coordinator.queueSize}
|
||||
}
|
||||
record := UploadRunRecord{
|
||||
ID: runID,
|
||||
PipelineID: pipeline.ID,
|
||||
Status: UploadStatusAccepted,
|
||||
AcceptedAt: coordinator.now().UTC(),
|
||||
}
|
||||
coordinator.records[runID] = record
|
||||
coordinator.pending = append(coordinator.pending, &uploadJob{
|
||||
recordID: runID,
|
||||
request: request,
|
||||
pipeline: pipeline,
|
||||
})
|
||||
coordinator.notify()
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
coordinator.expireLocked(coordinator.now().UTC())
|
||||
record, ok := coordinator.records[runID]
|
||||
return record, ok
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) Expire() []UploadRunRecord {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
return coordinator.expireLocked(coordinator.now().UTC())
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) CanAccept() bool {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
coordinator.expireLocked(coordinator.now().UTC())
|
||||
return len(coordinator.pending) < coordinator.queueSize
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) QueueDepth() int {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
return len(coordinator.pending)
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) RunningCount() int {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
return coordinator.runningCount
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) newRunID(pipelineID string) (UploadRunID, error) {
|
||||
suffix, err := coordinator.randomSuffix()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
timestamp := coordinator.now().UTC().Format("20060102T150405Z")
|
||||
return UploadRunID(pipelineID + "." + timestamp + "." + suffix), nil
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) dispatchLoop() {
|
||||
for {
|
||||
select {
|
||||
case <-coordinator.ctx.Done():
|
||||
return
|
||||
case <-coordinator.signal:
|
||||
for coordinator.startNext() {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) startNext() bool {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
if coordinator.runningCount >= coordinator.maxConcurrency {
|
||||
coordinator.markPendingQueuedLocked()
|
||||
return false
|
||||
}
|
||||
index := -1
|
||||
for candidateIndex, job := range coordinator.pending {
|
||||
if coordinator.activePipeline[job.pipeline.ID] {
|
||||
record := coordinator.records[job.recordID]
|
||||
if record.Status == UploadStatusAccepted {
|
||||
record.Status = UploadStatusQueued
|
||||
coordinator.records[job.recordID] = record
|
||||
}
|
||||
continue
|
||||
}
|
||||
index = candidateIndex
|
||||
break
|
||||
}
|
||||
if index < 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
job := coordinator.pending[index]
|
||||
coordinator.pending = append(coordinator.pending[:index], coordinator.pending[index+1:]...)
|
||||
now := coordinator.now().UTC()
|
||||
record := coordinator.records[job.recordID]
|
||||
record.Status = UploadStatusRunning
|
||||
record.StartedAt = &now
|
||||
coordinator.records[job.recordID] = record
|
||||
coordinator.runningCount++
|
||||
coordinator.activePipeline[job.pipeline.ID] = true
|
||||
|
||||
go coordinator.runJob(job)
|
||||
return true
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) markPendingQueuedLocked() {
|
||||
for _, job := range coordinator.pending {
|
||||
record := coordinator.records[job.recordID]
|
||||
if record.Status == UploadStatusAccepted {
|
||||
record.Status = UploadStatusQueued
|
||||
coordinator.records[job.recordID] = record
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) runJob(job *uploadJob) {
|
||||
record := coordinator.currentRecord(job.recordID)
|
||||
maxFileCount := job.request.MaxFileCount
|
||||
if maxFileCount <= 0 {
|
||||
maxFileCount = DefaultUploadMaxFileCount
|
||||
}
|
||||
staged, err := coordinator.stage(coordinator.ctx, ingest.StageOptions{
|
||||
Body: job.request.Body,
|
||||
ContentType: job.request.ContentType,
|
||||
PipelineStagingPath: job.pipeline.Source.Upload.StagingPath,
|
||||
RunID: string(record.ID),
|
||||
MaxUploadSize: int64(*job.pipeline.Source.Upload.MaxUploadSize),
|
||||
MaxExtractedSize: int64(*job.pipeline.Source.Upload.MaxUploadSize),
|
||||
MaxFileCount: maxFileCount,
|
||||
})
|
||||
if err == nil {
|
||||
coordinator.setStagedRoot(job.recordID, staged.Root)
|
||||
var report RunReport
|
||||
report, err = coordinator.run(coordinator.ctx, coordinator.cfg, RunPipelineWithLocalSourceOptions{
|
||||
PipelineID: job.pipeline.ID,
|
||||
SourceRoot: staged.Root,
|
||||
DryRun: job.request.DryRun,
|
||||
Force: job.request.Force,
|
||||
})
|
||||
coordinator.complete(job, &report, err)
|
||||
return
|
||||
}
|
||||
coordinator.complete(job, nil, err)
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) currentRecord(runID UploadRunID) UploadRunRecord {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
return coordinator.records[runID]
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) setStagedRoot(runID UploadRunID, root string) {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
record := coordinator.records[runID]
|
||||
record.StagedRoot = root
|
||||
coordinator.records[runID] = record
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) complete(job *uploadJob, report *RunReport, runErr error) {
|
||||
coordinator.mu.Lock()
|
||||
defer coordinator.mu.Unlock()
|
||||
record := coordinator.records[job.recordID]
|
||||
finishedAt := coordinator.now().UTC()
|
||||
record.FinishedAt = &finishedAt
|
||||
record.Report = report
|
||||
if runErr != nil {
|
||||
record.Status = UploadStatusFailed
|
||||
record.Error = runErr.Error()
|
||||
} else {
|
||||
record.Status = UploadStatusSucceeded
|
||||
}
|
||||
coordinator.records[job.recordID] = record
|
||||
coordinator.runningCount--
|
||||
delete(coordinator.activePipeline, job.pipeline.ID)
|
||||
coordinator.notify()
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) expireLocked(now time.Time) []UploadRunRecord {
|
||||
var expired []UploadRunRecord
|
||||
for runID, record := range coordinator.records {
|
||||
if record.FinishedAt == nil || record.Status == UploadStatusExpired {
|
||||
continue
|
||||
}
|
||||
if now.Before(record.FinishedAt.Add(coordinator.retention)) {
|
||||
continue
|
||||
}
|
||||
if record.StagedRoot != "" {
|
||||
_ = os.RemoveAll(record.StagedRoot)
|
||||
}
|
||||
record.Status = UploadStatusExpired
|
||||
record.Report = nil
|
||||
record.Error = ""
|
||||
expired = append(expired, record)
|
||||
delete(coordinator.records, runID)
|
||||
}
|
||||
return expired
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) notify() {
|
||||
select {
|
||||
case coordinator.signal <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func randomRunIDSuffix() (string, error) {
|
||||
var data [4]byte
|
||||
if _, err := rand.Read(data[:]); err != nil {
|
||||
return "", fmt.Errorf("generate run id suffix: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(data[:]), nil
|
||||
}
|
||||
373
internal/app/upload_coordinator_test.go
Normal file
373
internal/app/upload_coordinator_test.go
Normal file
@@ -0,0 +1,373 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
|
||||
)
|
||||
|
||||
func TestUploadCoordinatorGeneratesRunIDAndAcceptedStatus(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
clock := newUploadTestClock(time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC))
|
||||
coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
|
||||
pipelineIDs: []string{"weather-daily"},
|
||||
}), uploadCoordinatorHooks{
|
||||
now: clock.Now,
|
||||
randomSuffix: uploadTestSuffixes("ab12cd34"),
|
||||
stage: successfulUploadStage,
|
||||
run: successfulUploadRun,
|
||||
})
|
||||
|
||||
record, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
PipelineID: "weather-daily",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("archive"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Submit() error = %v", err)
|
||||
}
|
||||
if got, want := record.ID, UploadRunID("weather-daily.20260603T120000Z.ab12cd34"); got != want {
|
||||
t.Fatalf("run id = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := record.Status, UploadStatusAccepted; got != want {
|
||||
t.Fatalf("initial status = %q, want %q", got, want)
|
||||
}
|
||||
waitForUploadStatus(t, coordinator, record.ID, UploadStatusSucceeded)
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorRejectsFullQueueBeforeReadingBody(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
release := make(chan struct{})
|
||||
var reads atomic.Int64
|
||||
coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
|
||||
pipelineIDs: []string{"reports"},
|
||||
queueSize: 1,
|
||||
maxConcurrency: 1,
|
||||
}), uploadCoordinatorHooks{
|
||||
randomSuffix: uploadTestSuffixes("00000001", "00000002", "00000003"),
|
||||
stage: successfulUploadStage,
|
||||
run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
<-release
|
||||
return RunReport{}, nil
|
||||
},
|
||||
})
|
||||
|
||||
first, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("first"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("first Submit() error = %v", err)
|
||||
}
|
||||
waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
|
||||
|
||||
second, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("second"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("second Submit() error = %v", err)
|
||||
}
|
||||
waitForUploadStatus(t, coordinator, second.ID, UploadStatusQueued)
|
||||
|
||||
_, err = coordinator.Submit(context.Background(), UploadRequest{
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: readerFunc(func(data []byte) (int, error) {
|
||||
reads.Add(1)
|
||||
return 0, io.EOF
|
||||
}),
|
||||
})
|
||||
if err == nil || !IsUploadQueueFull(err) {
|
||||
t.Fatalf("third Submit() error = %v, want full queue", err)
|
||||
}
|
||||
if got := reads.Load(); got != 0 {
|
||||
t.Fatalf("rejected body reads = %d, want 0", got)
|
||||
}
|
||||
close(release)
|
||||
waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
|
||||
waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorSerializesSamePipelineUploads(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
release := make(chan struct{})
|
||||
coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
|
||||
pipelineIDs: []string{"reports"},
|
||||
queueSize: 4,
|
||||
maxConcurrency: 2,
|
||||
}), uploadCoordinatorHooks{
|
||||
randomSuffix: uploadTestSuffixes("00000001", "00000002"),
|
||||
stage: successfulUploadStage,
|
||||
run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
<-release
|
||||
return RunReport{}, nil
|
||||
},
|
||||
})
|
||||
|
||||
first, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("first")})
|
||||
if err != nil {
|
||||
t.Fatalf("first Submit() error = %v", err)
|
||||
}
|
||||
second, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("second")})
|
||||
if err != nil {
|
||||
t.Fatalf("second Submit() error = %v", err)
|
||||
}
|
||||
waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
|
||||
waitForUploadStatus(t, coordinator, second.ID, UploadStatusQueued)
|
||||
if got := coordinator.RunningCount(); got != 1 {
|
||||
t.Fatalf("running count = %d, want 1", got)
|
||||
}
|
||||
close(release)
|
||||
waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
|
||||
waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorRunsDifferentPipelinesConcurrentlyUpToLimit(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
release := make(chan struct{})
|
||||
coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
|
||||
pipelineIDs: []string{"reports-one", "reports-two"},
|
||||
queueSize: 4,
|
||||
maxConcurrency: 2,
|
||||
}), uploadCoordinatorHooks{
|
||||
randomSuffix: uploadTestSuffixes("00000001", "00000002"),
|
||||
stage: successfulUploadStage,
|
||||
run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
<-release
|
||||
return RunReport{}, nil
|
||||
},
|
||||
})
|
||||
|
||||
first, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports-one", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("first")})
|
||||
if err != nil {
|
||||
t.Fatalf("first Submit() error = %v", err)
|
||||
}
|
||||
second, err := coordinator.Submit(context.Background(), UploadRequest{PipelineID: "reports-two", ContentType: ingest.ContentTypeTar, Body: strings.NewReader("second")})
|
||||
if err != nil {
|
||||
t.Fatalf("second Submit() error = %v", err)
|
||||
}
|
||||
waitForUploadStatus(t, coordinator, first.ID, UploadStatusRunning)
|
||||
waitForUploadStatus(t, coordinator, second.ID, UploadStatusRunning)
|
||||
if got := coordinator.RunningCount(); got != 2 {
|
||||
t.Fatalf("running count = %d, want 2", got)
|
||||
}
|
||||
close(release)
|
||||
waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
|
||||
waitForUploadStatus(t, coordinator, second.ID, UploadStatusSucceeded)
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorRecordsFailureDetails(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
runErr := errors.New("publish failed")
|
||||
coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
|
||||
pipelineIDs: []string{"reports"},
|
||||
}), uploadCoordinatorHooks{
|
||||
randomSuffix: uploadTestSuffixes("00000001"),
|
||||
stage: successfulUploadStage,
|
||||
run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
return RunReport{DryRun: options.DryRun}, runErr
|
||||
},
|
||||
})
|
||||
|
||||
record, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("archive"),
|
||||
DryRun: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Submit() error = %v", err)
|
||||
}
|
||||
failed := waitForUploadStatus(t, coordinator, record.ID, UploadStatusFailed)
|
||||
if failed.Error != runErr.Error() {
|
||||
t.Fatalf("error = %q, want %q", failed.Error, runErr.Error())
|
||||
}
|
||||
if failed.Report == nil || !failed.Report.DryRun {
|
||||
t.Fatalf("report = %#v, want retained dry-run report", failed.Report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorExpiresCompletedRecordsAndStagingDirectories(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
clock := newUploadTestClock(time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC))
|
||||
coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
|
||||
pipelineIDs: []string{"reports"},
|
||||
retention: time.Second,
|
||||
}), uploadCoordinatorHooks{
|
||||
now: clock.Now,
|
||||
randomSuffix: uploadTestSuffixes("00000001"),
|
||||
stage: successfulUploadStage,
|
||||
run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
return RunReport{DryRun: true}, nil
|
||||
},
|
||||
})
|
||||
|
||||
record, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("archive"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Submit() error = %v", err)
|
||||
}
|
||||
succeeded := waitForUploadStatus(t, coordinator, record.ID, UploadStatusSucceeded)
|
||||
if succeeded.Report == nil || !succeeded.Report.DryRun {
|
||||
t.Fatalf("report = %#v, want retained dry-run report", succeeded.Report)
|
||||
}
|
||||
if _, err := os.Stat(succeeded.StagedRoot); err != nil {
|
||||
t.Fatalf("staged root stat before expiry = %v", err)
|
||||
}
|
||||
|
||||
clock.Advance(2 * time.Second)
|
||||
expired := coordinator.Expire()
|
||||
if got, want := len(expired), 1; got != want {
|
||||
t.Fatalf("expired count = %d, want %d", got, want)
|
||||
}
|
||||
if expired[0].Status != UploadStatusExpired || expired[0].Report != nil || expired[0].Error != "" {
|
||||
t.Fatalf("expired record = %#v, want expired without report/error", expired[0])
|
||||
}
|
||||
if _, ok := coordinator.Status(record.ID); ok {
|
||||
t.Fatal("Status() ok = true after expiry, want removed status")
|
||||
}
|
||||
if _, err := os.Stat(succeeded.StagedRoot); !os.IsNotExist(err) {
|
||||
t.Fatalf("staged root stat after expiry = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
type readerFunc func([]byte) (int, error)
|
||||
|
||||
func (fn readerFunc) Read(data []byte) (int, error) {
|
||||
return fn(data)
|
||||
}
|
||||
|
||||
func successfulUploadStage(ctx context.Context, opts ingest.StageOptions) (ingest.StagedBundle, error) {
|
||||
root := filepath.Join(opts.PipelineStagingPath, opts.RunID)
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return ingest.StagedBundle{}, err
|
||||
}
|
||||
return ingest.StagedBundle{Root: root}, nil
|
||||
}
|
||||
|
||||
func successfulUploadRun(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
return RunReport{}, nil
|
||||
}
|
||||
|
||||
type uploadCoordinatorConfigOptions struct {
|
||||
pipelineIDs []string
|
||||
queueSize int
|
||||
maxConcurrency int
|
||||
retention time.Duration
|
||||
}
|
||||
|
||||
func uploadCoordinatorConfig(t *testing.T, opts uploadCoordinatorConfigOptions) config.Config {
|
||||
t.Helper()
|
||||
queueSize := opts.queueSize
|
||||
if queueSize == 0 {
|
||||
queueSize = 4
|
||||
}
|
||||
maxConcurrency := opts.maxConcurrency
|
||||
if maxConcurrency == 0 {
|
||||
maxConcurrency = 1
|
||||
}
|
||||
retentionValue := opts.retention
|
||||
if retentionValue == 0 {
|
||||
retentionValue = time.Minute
|
||||
}
|
||||
retention := config.Duration(retentionValue)
|
||||
maxUploadSize := config.ByteSize(1024)
|
||||
cfg := config.Config{
|
||||
Server: config.Server{HTTP: config.HTTPServer{
|
||||
StagingRoot: t.TempDir(),
|
||||
MaxUploadSize: &maxUploadSize,
|
||||
QueueSize: queueSize,
|
||||
MaxConcurrency: maxConcurrency,
|
||||
Retention: &retention,
|
||||
}},
|
||||
}
|
||||
for _, pipelineID := range opts.pipelineIDs {
|
||||
cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
|
||||
ID: pipelineID,
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{TokenEnv: strings.ToUpper(strings.ReplaceAll(pipelineID, "-", "_")) + "_TOKEN"},
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "archive",
|
||||
Backend: config.BackendLocal,
|
||||
Path: t.TempDir(),
|
||||
}},
|
||||
})
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
type uploadTestClock struct {
|
||||
mu sync.Mutex
|
||||
now time.Time
|
||||
}
|
||||
|
||||
func newUploadTestClock(now time.Time) *uploadTestClock {
|
||||
return &uploadTestClock{now: now}
|
||||
}
|
||||
|
||||
func (clock *uploadTestClock) Now() time.Time {
|
||||
clock.mu.Lock()
|
||||
defer clock.mu.Unlock()
|
||||
return clock.now
|
||||
}
|
||||
|
||||
func (clock *uploadTestClock) Advance(duration time.Duration) {
|
||||
clock.mu.Lock()
|
||||
defer clock.mu.Unlock()
|
||||
clock.now = clock.now.Add(duration)
|
||||
}
|
||||
|
||||
func uploadTestSuffixes(values ...string) func() (string, error) {
|
||||
var mu sync.Mutex
|
||||
index := 0
|
||||
return func() (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if index >= len(values) {
|
||||
return fmt.Sprintf("%08d", index+1), nil
|
||||
}
|
||||
value := values[index]
|
||||
index++
|
||||
return value, nil
|
||||
}
|
||||
}
|
||||
|
||||
func waitForUploadStatus(t *testing.T, coordinator *UploadCoordinator, runID UploadRunID, status UploadStatus) UploadRunRecord {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
record, ok := coordinator.Status(runID)
|
||||
if ok && record.Status == status {
|
||||
return record
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
record, ok := coordinator.Status(runID)
|
||||
t.Fatalf("timed out waiting for status %s; latest ok=%t record=%#v", status, ok, record)
|
||||
return UploadRunRecord{}
|
||||
}
|
||||
209
internal/app/upload_http.go
Normal file
209
internal/app/upload_http.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
|
||||
)
|
||||
|
||||
type uploadCoordinator interface {
|
||||
CanAccept() bool
|
||||
Submit(context.Context, UploadRequest) (UploadRunRecord, error)
|
||||
Status(UploadRunID) (UploadRunRecord, bool)
|
||||
}
|
||||
|
||||
type uploadHTTPHandler struct {
|
||||
coordinator uploadCoordinator
|
||||
tokens map[string]string
|
||||
limits map[string]int64
|
||||
}
|
||||
|
||||
type uploadAcceptedResponse struct {
|
||||
RunID UploadRunID `json:"run_id"`
|
||||
Status UploadStatus `json:"status"`
|
||||
}
|
||||
|
||||
type httpErrorResponse struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
func newUploadHTTPHandler(ctx context.Context, cfg config.Config, environment config.Environment) (http.Handler, error) {
|
||||
config.ApplyDefaults(&cfg)
|
||||
tokens, limits, err := resolveUploadTokens(cfg, environment)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return uploadHTTPHandler{
|
||||
coordinator: NewUploadCoordinator(ctx, cfg),
|
||||
tokens: tokens,
|
||||
limits: limits,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]string, map[string]int64, error) {
|
||||
tokens := make(map[string]string)
|
||||
limits := make(map[string]int64)
|
||||
for _, pipeline := range cfg.Pipelines {
|
||||
if pipeline.Source.Backend != config.BackendHTTPUpload {
|
||||
continue
|
||||
}
|
||||
tokenName := pipeline.Source.Upload.TokenEnv
|
||||
token, ok := environment.Lookup(tokenName)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("upload token environment variable %s is not set", tokenName)
|
||||
}
|
||||
if token == "" {
|
||||
return nil, nil, fmt.Errorf("upload token environment variable %s is empty", tokenName)
|
||||
}
|
||||
if existing, exists := tokens[token]; exists {
|
||||
return nil, nil, fmt.Errorf("upload token environment variables for pipelines %s and %s resolve to the same value", existing, pipeline.ID)
|
||||
}
|
||||
tokens[token] = pipeline.ID
|
||||
limits[pipeline.ID] = int64(*pipeline.Source.Upload.MaxUploadSize)
|
||||
}
|
||||
return tokens, limits, nil
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/healthz":
|
||||
handler.handleHealth(w)
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/upload":
|
||||
handler.handleUpload(w, r)
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/runs/"):
|
||||
handler.handleRunStatus(w, r)
|
||||
default:
|
||||
writeHTTPError(w, http.StatusNotFound, "not found")
|
||||
}
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) handleHealth(w http.ResponseWriter) {
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Has("pipeline") || r.URL.Query().Has("pipeline_id") {
|
||||
writeHTTPError(w, http.StatusBadRequest, "pipeline id is not accepted")
|
||||
return
|
||||
}
|
||||
pipelineID, ok := handler.authenticate(r.Header.Get("Authorization"))
|
||||
if !ok {
|
||||
writeHTTPError(w, http.StatusUnauthorized, "unauthorized")
|
||||
return
|
||||
}
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if !supportedUploadContentType(contentType) {
|
||||
writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
|
||||
return
|
||||
}
|
||||
if !handler.coordinator.CanAccept() {
|
||||
writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
|
||||
return
|
||||
}
|
||||
body, err := readUploadBody(r.Body, handler.limits[pipelineID])
|
||||
if err != nil {
|
||||
if errors.Is(err, ingest.ErrUploadTooLarge) {
|
||||
writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
|
||||
return
|
||||
}
|
||||
writeHTTPError(w, http.StatusBadRequest, "read upload body failed")
|
||||
return
|
||||
}
|
||||
record, err := handler.coordinator.Submit(r.Context(), UploadRequest{
|
||||
PipelineID: pipelineID,
|
||||
ContentType: contentType,
|
||||
Body: bytes.NewReader(body),
|
||||
})
|
||||
if err != nil {
|
||||
writeUploadSubmitError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, uploadAcceptedResponse{
|
||||
RunID: record.ID,
|
||||
Status: UploadStatusAccepted,
|
||||
})
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) handleRunStatus(w http.ResponseWriter, r *http.Request) {
|
||||
rawRunID := strings.TrimPrefix(r.URL.Path, "/runs/")
|
||||
if rawRunID == "" || strings.Contains(rawRunID, "/") {
|
||||
writeHTTPError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
record, ok := handler.coordinator.Status(UploadRunID(rawRunID))
|
||||
if !ok {
|
||||
writeHTTPError(w, http.StatusNotFound, "run not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, record)
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) authenticate(header string) (string, bool) {
|
||||
const prefix = "Bearer "
|
||||
if !strings.HasPrefix(header, prefix) {
|
||||
return "", false
|
||||
}
|
||||
token := strings.TrimSpace(strings.TrimPrefix(header, prefix))
|
||||
if token == "" {
|
||||
return "", false
|
||||
}
|
||||
pipelineID, ok := handler.tokens[token]
|
||||
return pipelineID, ok
|
||||
}
|
||||
|
||||
func supportedUploadContentType(contentType string) bool {
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
mediaType = contentType
|
||||
}
|
||||
switch mediaType {
|
||||
case ingest.ContentTypeTar, ingest.ContentTypeGzip, ingest.ContentTypeXGzip:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func readUploadBody(body io.Reader, maxSize int64) ([]byte, error) {
|
||||
limited := &io.LimitedReader{R: body, N: maxSize + 1}
|
||||
data, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > maxSize {
|
||||
return nil, ingest.ErrUploadTooLarge
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func writeUploadSubmitError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case IsUploadQueueFull(err):
|
||||
writeHTTPError(w, http.StatusServiceUnavailable, "upload queue is full")
|
||||
case errors.Is(err, ingest.ErrUploadTooLarge):
|
||||
writeHTTPError(w, http.StatusRequestEntityTooLarge, "upload exceeds maximum size")
|
||||
case errors.Is(err, ingest.ErrUnsupportedContentType):
|
||||
writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
|
||||
default:
|
||||
writeHTTPError(w, http.StatusBadRequest, "upload rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func writeHTTPError(w http.ResponseWriter, status int, message string) {
|
||||
writeJSON(w, status, httpErrorResponse{Error: message})
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
419
internal/app/upload_http_integration_test.go
Normal file
419
internal/app/upload_http_integration_test.go
Normal file
@@ -0,0 +1,419 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestHTTPUploadPublishesTarAndGzipFanout(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
compressed bool
|
||||
contentType string
|
||||
}{
|
||||
{name: "tar", contentType: ingest.ContentTypeTar},
|
||||
{name: "gzip", compressed: true, contentType: ingest.ContentTypeGzip},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
|
||||
id: "reports",
|
||||
tokenEnv: "REPORTS_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports"),
|
||||
destinations: []string{firstDestination, secondDestination},
|
||||
}}, 4, 1)
|
||||
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"REPORTS_TOKEN": "reports-secret",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("newUploadHTTPHandler() error = %v", err)
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
runID := submitHTTPUpload(t, server, "reports-secret", tt.contentType, bundleArchive(t, tt.compressed, testutil.BundleOptions{}))
|
||||
record := waitForHTTPUploadStatus(t, server, runID, UploadStatusSucceeded)
|
||||
|
||||
if record.Report == nil {
|
||||
t.Fatal("completed status report = nil, want run report")
|
||||
}
|
||||
if record.Report.Summary.Status != "ok" {
|
||||
t.Fatalf("summary status = %q, want ok", record.Report.Summary.Status)
|
||||
}
|
||||
if got, want := len(record.Report.Actions), 2; got != want {
|
||||
t.Fatalf("action count = %d, want %d", got, want)
|
||||
}
|
||||
assertPublishedBundle(t, firstDestination)
|
||||
assertPublishedBundle(t, secondDestination)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPUploadInvalidArchiveFailsWithoutPublishing(t *testing.T) {
|
||||
destination := t.TempDir()
|
||||
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
|
||||
id: "reports",
|
||||
tokenEnv: "REPORTS_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports"),
|
||||
destinations: []string{destination},
|
||||
}}, 4, 1)
|
||||
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"REPORTS_TOKEN": "reports-secret",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("newUploadHTTPHandler() error = %v", err)
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
runID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("not a tar archive"))
|
||||
record := waitForHTTPUploadStatus(t, server, runID, UploadStatusFailed)
|
||||
|
||||
if record.Error == "" {
|
||||
t.Fatal("failed status error is empty")
|
||||
}
|
||||
if record.Report != nil {
|
||||
t.Fatalf("failed staging report = %#v, want nil", record.Report)
|
||||
}
|
||||
assertDirectoryEmpty(t, destination)
|
||||
}
|
||||
|
||||
func TestHTTPUploadSamePipelineRequestsSerialize(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
release := make(chan struct{})
|
||||
started := make(chan struct{}, 1)
|
||||
coordinator := newUploadCoordinator(ctx, httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
|
||||
id: "reports",
|
||||
tokenEnv: "REPORTS_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports"),
|
||||
destinations: []string{t.TempDir()},
|
||||
}}, 4, 2), uploadCoordinatorHooks{
|
||||
randomSuffix: uploadTestSuffixes("00000001", "00000002"),
|
||||
stage: successfulUploadStage,
|
||||
run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
select {
|
||||
case started <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-release
|
||||
return RunReport{}, nil
|
||||
},
|
||||
})
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]string{"reports-secret": "reports"},
|
||||
limits: map[string]int64{"reports": 1024},
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
firstRunID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("first"))
|
||||
waitForRunStart(t, started)
|
||||
first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusRunning)
|
||||
secondRunID := submitHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("second"))
|
||||
second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusQueued)
|
||||
|
||||
if first.PipelineID != "reports" || second.PipelineID != "reports" {
|
||||
t.Fatalf("statuses = %#v %#v, want same pipeline", first, second)
|
||||
}
|
||||
if got := coordinator.RunningCount(); got != 1 {
|
||||
t.Fatalf("running count = %d, want 1", got)
|
||||
}
|
||||
close(release)
|
||||
waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
|
||||
waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
|
||||
}
|
||||
|
||||
func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
release := make(chan struct{})
|
||||
started := make(chan string, 2)
|
||||
coordinator := newUploadCoordinator(ctx, httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
|
||||
{
|
||||
id: "reports-one",
|
||||
tokenEnv: "REPORTS_ONE_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports-one"),
|
||||
destinations: []string{t.TempDir()},
|
||||
},
|
||||
{
|
||||
id: "reports-two",
|
||||
tokenEnv: "REPORTS_TWO_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports-two"),
|
||||
destinations: []string{t.TempDir()},
|
||||
},
|
||||
}, 4, 2), uploadCoordinatorHooks{
|
||||
randomSuffix: uploadTestSuffixes("00000001", "00000002"),
|
||||
stage: successfulUploadStage,
|
||||
run: func(ctx context.Context, cfg config.Config, options RunPipelineWithLocalSourceOptions) (RunReport, error) {
|
||||
started <- options.PipelineID
|
||||
<-release
|
||||
return RunReport{}, nil
|
||||
},
|
||||
})
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]string{
|
||||
"one-secret": "reports-one",
|
||||
"two-secret": "reports-two",
|
||||
},
|
||||
limits: map[string]int64{
|
||||
"reports-one": 1024,
|
||||
"reports-two": 1024,
|
||||
},
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
firstRunID := submitHTTPUpload(t, server, "one-secret", ingest.ContentTypeTar, []byte("first"))
|
||||
secondRunID := submitHTTPUpload(t, server, "two-secret", ingest.ContentTypeTar, []byte("second"))
|
||||
waitForStartedPipelines(t, started, "reports-one", "reports-two")
|
||||
waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusRunning)
|
||||
waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusRunning)
|
||||
|
||||
if got := coordinator.RunningCount(); got != 2 {
|
||||
t.Fatalf("running count = %d, want 2", got)
|
||||
}
|
||||
close(release)
|
||||
waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
|
||||
waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
|
||||
}
|
||||
|
||||
type httpUploadPipelineSpec struct {
|
||||
id string
|
||||
tokenEnv string
|
||||
stagingPath string
|
||||
destinations []string
|
||||
}
|
||||
|
||||
func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpec, queueSize, maxConcurrency int) config.Config {
|
||||
t.Helper()
|
||||
size := config.ByteSize(1024 * 1024)
|
||||
retention := config.Duration(time.Minute)
|
||||
cfg := config.Config{
|
||||
Server: config.Server{HTTP: config.HTTPServer{
|
||||
Bind: config.DefaultHTTPBind,
|
||||
StagingRoot: t.TempDir(),
|
||||
MaxUploadSize: &size,
|
||||
QueueSize: queueSize,
|
||||
MaxConcurrency: maxConcurrency,
|
||||
Retention: &retention,
|
||||
}},
|
||||
}
|
||||
for _, spec := range pipelines {
|
||||
pipeline := config.Pipeline{
|
||||
ID: spec.id,
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{
|
||||
TokenEnv: spec.tokenEnv,
|
||||
StagingPath: spec.stagingPath,
|
||||
MaxUploadSize: &size,
|
||||
},
|
||||
},
|
||||
}
|
||||
for index, destination := range spec.destinations {
|
||||
pipeline.Destinations = append(pipeline.Destinations, config.Destination{
|
||||
ID: fmt.Sprintf("archive-%d", index+1),
|
||||
Backend: config.BackendLocal,
|
||||
Path: destination,
|
||||
Publish: &config.PublishPolicy{Source: true},
|
||||
})
|
||||
}
|
||||
cfg.Pipelines = append(cfg.Pipelines, pipeline)
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) UploadRunID {
|
||||
t.Helper()
|
||||
request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("NewRequest() error = %v", err)
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
request.Header.Set("Content-Type", contentType)
|
||||
response, err := server.Client().Do(request)
|
||||
if err != nil {
|
||||
t.Fatalf("POST /upload error = %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusAccepted {
|
||||
t.Fatalf("POST /upload status = %d, want %d", response.StatusCode, http.StatusAccepted)
|
||||
}
|
||||
var accepted uploadAcceptedResponse
|
||||
if err := json.NewDecoder(response.Body).Decode(&accepted); err != nil {
|
||||
t.Fatalf("decode accepted response: %v", err)
|
||||
}
|
||||
if accepted.RunID == "" || accepted.Status != UploadStatusAccepted {
|
||||
t.Fatalf("accepted response = %#v, want run id and accepted status", accepted)
|
||||
}
|
||||
return accepted.RunID
|
||||
}
|
||||
|
||||
func waitForHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID, status UploadStatus) UploadRunRecord {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
var latest UploadRunRecord
|
||||
var latestStatus int
|
||||
for time.Now().Before(deadline) {
|
||||
latest, latestStatus = getHTTPUploadStatus(t, server, runID)
|
||||
if latestStatus == http.StatusOK && latest.Status == status {
|
||||
return latest
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out waiting for status %s; latest HTTP status=%d record=%#v", status, latestStatus, latest)
|
||||
return UploadRunRecord{}
|
||||
}
|
||||
|
||||
func getHTTPUploadStatus(t *testing.T, server *httptest.Server, runID UploadRunID) (UploadRunRecord, int) {
|
||||
t.Helper()
|
||||
response, err := server.Client().Get(server.URL + "/runs/" + string(runID))
|
||||
if err != nil {
|
||||
t.Fatalf("GET /runs error = %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return UploadRunRecord{}, response.StatusCode
|
||||
}
|
||||
var record UploadRunRecord
|
||||
if err := json.NewDecoder(response.Body).Decode(&record); err != nil {
|
||||
t.Fatalf("decode run status: %v", err)
|
||||
}
|
||||
return record, response.StatusCode
|
||||
}
|
||||
|
||||
func bundleArchive(t *testing.T, compressed bool, opts testutil.BundleOptions) []byte {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, root, "", opts)
|
||||
return tarDirectory(t, root, compressed)
|
||||
}
|
||||
|
||||
func tarDirectory(t *testing.T, root string, compressed bool) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
var writer io.WriteCloser = nopWriteCloser{writer: &output}
|
||||
if compressed {
|
||||
gzipWriter := gzip.NewWriter(&output)
|
||||
writer = gzipWriter
|
||||
}
|
||||
tarWriter := tar.NewWriter(writer)
|
||||
if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
relative, err := filepath.Rel(root, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header := &tar.Header{
|
||||
Name: filepath.ToSlash(relative),
|
||||
Mode: 0o600,
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
if err := tarWriter.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tarWriter.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("walk bundle: %v", err)
|
||||
}
|
||||
if err := tarWriter.Close(); err != nil {
|
||||
t.Fatalf("close tar: %v", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close archive: %v", err)
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
type nopWriteCloser struct {
|
||||
writer io.Writer
|
||||
}
|
||||
|
||||
func (writer nopWriteCloser) Write(data []byte) (int, error) {
|
||||
return writer.writer.Write(data)
|
||||
}
|
||||
|
||||
func (writer nopWriteCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func assertPublishedBundle(t *testing.T, destinationRoot string) {
|
||||
t.Helper()
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
|
||||
t.Fatalf("destination state stat: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertDirectoryEmpty(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir() error = %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("directory %s has %d entries, want empty", root, len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func waitForRunStart(t *testing.T, started <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for run start")
|
||||
}
|
||||
}
|
||||
|
||||
func waitForStartedPipelines(t *testing.T, started <-chan string, want ...string) {
|
||||
t.Helper()
|
||||
remaining := map[string]bool{}
|
||||
for _, pipelineID := range want {
|
||||
remaining[pipelineID] = true
|
||||
}
|
||||
deadline := time.After(time.Second)
|
||||
for len(remaining) > 0 {
|
||||
select {
|
||||
case pipelineID := <-started:
|
||||
delete(remaining, pipelineID)
|
||||
case <-deadline:
|
||||
t.Fatalf("timed out waiting for pipelines to start; remaining=%v", remaining)
|
||||
}
|
||||
}
|
||||
}
|
||||
331
internal/app/upload_http_test.go
Normal file
331
internal/app/upload_http_test.go
Normal file
@@ -0,0 +1,331 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
type fakeUploadCoordinator struct {
|
||||
canAccept bool
|
||||
submit func(context.Context, UploadRequest) (UploadRunRecord, error)
|
||||
status func(UploadRunID) (UploadRunRecord, bool)
|
||||
}
|
||||
|
||||
func (fake fakeUploadCoordinator) CanAccept() bool {
|
||||
return fake.canAccept
|
||||
}
|
||||
|
||||
func (fake fakeUploadCoordinator) Submit(ctx context.Context, request UploadRequest) (UploadRunRecord, error) {
|
||||
if fake.submit == nil {
|
||||
return UploadRunRecord{}, errors.New("unexpected submit")
|
||||
}
|
||||
return fake.submit(ctx, request)
|
||||
}
|
||||
|
||||
func (fake fakeUploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
|
||||
if fake.status == nil {
|
||||
return UploadRunRecord{}, false
|
||||
}
|
||||
return fake.status(runID)
|
||||
}
|
||||
|
||||
func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) {
|
||||
cfg := uploadHTTPTestConfig()
|
||||
|
||||
_, _, err := resolveUploadTokens(cfg, config.NewEnvironment(nil, func(string) (string, bool) {
|
||||
return "", false
|
||||
}))
|
||||
if err == nil || !strings.Contains(err.Error(), "UPLOAD_TOKEN") {
|
||||
t.Fatalf("resolveUploadTokens() error = %v, want missing UPLOAD_TOKEN", err)
|
||||
}
|
||||
|
||||
cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
|
||||
ID: "weekly",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{TokenEnv: "OTHER_UPLOAD_TOKEN"},
|
||||
},
|
||||
Destinations: cfg.Pipelines[0].Destinations,
|
||||
})
|
||||
config.ApplyDefaults(&cfg)
|
||||
secret := "super-secret-token"
|
||||
_, _, err = resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"UPLOAD_TOKEN": secret,
|
||||
"OTHER_UPLOAD_TOKEN": secret,
|
||||
}))
|
||||
if err == nil {
|
||||
t.Fatal("resolveUploadTokens() error = nil, want duplicate token error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatalf("duplicate token error exposed secret value: %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewUploadHTTPHandlerAcceptsDefaultedConfig(t *testing.T) {
|
||||
cfg := uploadHTTPTestConfig()
|
||||
cfg.Server.HTTP.Bind = ""
|
||||
cfg.Server.HTTP.StagingRoot = ""
|
||||
cfg.Server.HTTP.MaxUploadSize = nil
|
||||
cfg.Server.HTTP.QueueSize = 0
|
||||
cfg.Server.HTTP.MaxConcurrency = 0
|
||||
cfg.Server.HTTP.Retention = nil
|
||||
cfg.Pipelines[0].Source.Upload.StagingPath = ""
|
||||
cfg.Pipelines[0].Source.Upload.MaxUploadSize = nil
|
||||
|
||||
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"UPLOAD_TOKEN": "secret",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("newUploadHTTPHandler() error = %v", err)
|
||||
}
|
||||
if handler == nil {
|
||||
t.Fatal("newUploadHTTPHandler() = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
|
||||
var submitted UploadRequest
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{
|
||||
canAccept: true,
|
||||
submit: func(_ context.Context, request UploadRequest) (UploadRunRecord, error) {
|
||||
submitted = request
|
||||
body, err := io.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read submitted body: %v", err)
|
||||
}
|
||||
if string(body) != "archive" {
|
||||
t.Fatalf("submitted body = %q, want archive", body)
|
||||
}
|
||||
return UploadRunRecord{ID: "reports.20260603T120000Z.abcdef12", Status: UploadStatusAccepted}, nil
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
limits: map[string]int64{"reports": 1024},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusAccepted, recorder.Body.String())
|
||||
}
|
||||
if submitted.PipelineID != "reports" {
|
||||
t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID)
|
||||
}
|
||||
var response uploadAcceptedResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if response.RunID != "reports.20260603T120000Z.abcdef12" || response.Status != UploadStatusAccepted {
|
||||
t.Fatalf("response = %#v, want accepted run id", response)
|
||||
}
|
||||
if strings.Contains(recorder.Body.String(), "valid-token") {
|
||||
t.Fatalf("response exposed token: %q", recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{canAccept: true},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
limits: map[string]int64{"reports": 1024},
|
||||
}
|
||||
|
||||
for _, authHeader := range []string{"", "Bearer wrong-token"} {
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", authHeader)
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("auth %q status = %d, want %d", authHeader, recorder.Code, http.StatusUnauthorized)
|
||||
}
|
||||
if strings.Contains(recorder.Body.String(), "valid-token") || strings.Contains(recorder.Body.String(), "wrong-token") {
|
||||
t.Fatalf("unauthorized response exposed token: %q", recorder.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRejectsUnsupportedOversizedFullQueueAndPipelineID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
canAccept bool
|
||||
url string
|
||||
contentType string
|
||||
body io.Reader
|
||||
wantStatus int
|
||||
}{
|
||||
{
|
||||
name: "unsupported content type",
|
||||
canAccept: true,
|
||||
url: "/upload",
|
||||
contentType: "application/zip",
|
||||
body: strings.NewReader("archive"),
|
||||
wantStatus: http.StatusUnsupportedMediaType,
|
||||
},
|
||||
{
|
||||
name: "oversized",
|
||||
canAccept: true,
|
||||
url: "/upload",
|
||||
contentType: "application/x-tar",
|
||||
body: strings.NewReader("too-large"),
|
||||
wantStatus: http.StatusRequestEntityTooLarge,
|
||||
},
|
||||
{
|
||||
name: "full queue",
|
||||
canAccept: false,
|
||||
url: "/upload",
|
||||
contentType: "application/x-tar",
|
||||
body: &countingReader{reader: strings.NewReader("archive")},
|
||||
wantStatus: http.StatusServiceUnavailable,
|
||||
},
|
||||
{
|
||||
name: "submitted pipeline id",
|
||||
canAccept: true,
|
||||
url: "/upload?pipeline_id=reports",
|
||||
contentType: "application/x-tar",
|
||||
body: strings.NewReader("archive"),
|
||||
wantStatus: http.StatusBadRequest,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{
|
||||
canAccept: tt.canAccept,
|
||||
submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
|
||||
t.Fatal("Submit should not be called")
|
||||
return UploadRunRecord{}, nil
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
limits: map[string]int64{"reports": 4},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, tt.url, tt.body)
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", tt.contentType)
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != tt.wantStatus {
|
||||
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
|
||||
}
|
||||
if reader, ok := tt.body.(*countingReader); ok && reader.reads != 0 {
|
||||
t.Fatalf("full queue read body %d time(s), want zero", reader.reads)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRunStatusAndHealth(t *testing.T) {
|
||||
finishedAt := time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC)
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{
|
||||
canAccept: true,
|
||||
status: func(runID UploadRunID) (UploadRunRecord, bool) {
|
||||
if runID != "reports.20260603T120000Z.abcdef12" {
|
||||
return UploadRunRecord{}, false
|
||||
}
|
||||
return UploadRunRecord{
|
||||
ID: runID,
|
||||
PipelineID: "reports",
|
||||
Status: UploadStatusSucceeded,
|
||||
FinishedAt: &finishedAt,
|
||||
}, true
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
limits: map[string]int64{"reports": 1024},
|
||||
}
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("health status = %d, want %d", recorder.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
recorder = httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/reports.20260603T120000Z.abcdef12", nil))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("run status = %d, want %d; body = %q", recorder.Code, http.StatusOK, recorder.Body.String())
|
||||
}
|
||||
var record UploadRunRecord
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil {
|
||||
t.Fatalf("decode run status: %v", err)
|
||||
}
|
||||
if record.ID != "reports.20260603T120000Z.abcdef12" || record.Status != UploadStatusSucceeded {
|
||||
t.Fatalf("record = %#v, want succeeded run status", record)
|
||||
}
|
||||
|
||||
recorder = httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/runs/unknown", nil))
|
||||
if recorder.Code != http.StatusNotFound {
|
||||
t.Fatalf("unknown run status = %d, want %d", recorder.Code, http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
reader io.Reader
|
||||
reads int
|
||||
}
|
||||
|
||||
func (reader *countingReader) Read(data []byte) (int, error) {
|
||||
reader.reads++
|
||||
return reader.reader.Read(data)
|
||||
}
|
||||
|
||||
func uploadHTTPTestConfig() config.Config {
|
||||
size := config.ByteSize(1024)
|
||||
retention := config.Duration(24 * time.Hour)
|
||||
cfg := config.Config{
|
||||
Server: config.Server{HTTP: config.HTTPServer{
|
||||
Bind: config.DefaultHTTPBind,
|
||||
StagingRoot: "/tmp/distributor-test",
|
||||
MaxUploadSize: &size,
|
||||
QueueSize: 2,
|
||||
MaxConcurrency: 1,
|
||||
Retention: &retention,
|
||||
}},
|
||||
Pipelines: []config.Pipeline{{
|
||||
ID: "reports",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{
|
||||
TokenEnv: "UPLOAD_TOKEN",
|
||||
StagingPath: "/tmp/distributor-test/reports",
|
||||
MaxUploadSize: &size,
|
||||
},
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "local",
|
||||
Backend: config.BackendLocal,
|
||||
Path: "/tmp/distributor-output",
|
||||
Publish: &config.PublishPolicy{Source: true},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func uploadHTTPTestEnvironment(values map[string]string) config.Environment {
|
||||
return config.NewEnvironment(values, func(string) (string, bool) {
|
||||
return "", false
|
||||
})
|
||||
}
|
||||
@@ -29,6 +29,8 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
return versionCommand(ctx, args[1:], stdout, stderr)
|
||||
case "run":
|
||||
return runCommand(ctx, args[1:], stdout, stderr)
|
||||
case "serve":
|
||||
return serveCommand(ctx, args[1:], stdout, stderr)
|
||||
case "validate":
|
||||
return validateCommand(ctx, args[1:], stdout, stderr)
|
||||
case "inspect":
|
||||
@@ -51,6 +53,7 @@ Usage:
|
||||
Commands:
|
||||
version Print version information
|
||||
run Run configured distribution pipelines
|
||||
serve Run the HTTP upload server
|
||||
validate Validate a source bundle or bundle tree
|
||||
inspect Inspect bundles or distributor state
|
||||
manifest Create source bundle manifests
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
@@ -92,6 +93,34 @@ func TestExecuteVersionJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteServeParsesConfig(t *testing.T) {
|
||||
originalServeApp := serveApp
|
||||
defer func() {
|
||||
serveApp = originalServeApp
|
||||
}()
|
||||
var gotOptions app.ServeOptions
|
||||
serveApp = func(_ context.Context, options app.ServeOptions) error {
|
||||
gotOptions = options
|
||||
return nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{"serve", "--config", "config.yml"}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if gotOptions.ConfigPath != "config.yml" {
|
||||
t.Fatalf("ConfigPath = %q, want config.yml", gotOptions.ConfigPath)
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRejectsInvalidFormat(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
@@ -603,8 +632,12 @@ func TestExecuteRunDryRun(t *testing.T) {
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=publish_new") {
|
||||
t.Fatalf("stdout = %q, want config summary", stdout.String())
|
||||
wantStdout := "Configured pipelines: 1\n" +
|
||||
"- pipeline=reports source=local bundles=1 destinations=archive\n" +
|
||||
" - bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt reason=\"destination state is absent\"\n" +
|
||||
"Final status: ok planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true fixed_path=0\n"
|
||||
if got := stdout.String(); got != wantStdout {
|
||||
t.Fatalf("stdout = %q, want %q", got, wantStdout)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
@@ -631,6 +664,14 @@ func TestExecuteRunJSONDryRun(t *testing.T) {
|
||||
if result["dry_run"] != true {
|
||||
t.Fatalf("result = %#v, want dry_run true", result)
|
||||
}
|
||||
pipelines, ok := result["pipelines"].([]any)
|
||||
if !ok || len(pipelines) != 1 {
|
||||
t.Fatalf("pipelines = %#v, want one pipeline", result["pipelines"])
|
||||
}
|
||||
pipeline, ok := pipelines[0].(map[string]any)
|
||||
if !ok || pipeline["id"] != "reports" || pipeline["source_backend"] != "local" || pipeline["bundle_count"] != float64(1) {
|
||||
t.Fatalf("pipeline = %#v, want reports/local summary", pipelines[0])
|
||||
}
|
||||
actions, ok := result["actions"].([]any)
|
||||
if !ok || len(actions) != 1 {
|
||||
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||
@@ -639,6 +680,14 @@ func TestExecuteRunJSONDryRun(t *testing.T) {
|
||||
if !ok || action["action"] != "publish_new" {
|
||||
t.Fatalf("action = %#v, want publish_new", actions[0])
|
||||
}
|
||||
outputs, ok := action["outputs"].([]any)
|
||||
if !ok || len(outputs) != 2 {
|
||||
t.Fatalf("outputs = %#v, want source outputs", action["outputs"])
|
||||
}
|
||||
summary, ok := result["summary"].(map[string]any)
|
||||
if !ok || summary["status"] != "ok" || summary["planned"] != float64(1) || summary["publish_new"] != float64(1) || summary["dry_run"] != true {
|
||||
t.Fatalf("summary = %#v, want ok dry-run publish counters", result["summary"])
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
|
||||
46
internal/cli/serve.go
Normal file
46
internal/cli/serve.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||
)
|
||||
|
||||
var serveApp = app.Serve
|
||||
|
||||
func serveCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
if hasHelp(args) {
|
||||
printServeHelp(stdout)
|
||||
return exitOK
|
||||
}
|
||||
|
||||
flags := flag.NewFlagSet("serve", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
configPath := flags.String("config", "", "path to config file")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if rejectPositionalArgs(stderr, "serve", flags.Args()) {
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
if err := serveApp(ctx, app.ServeOptions{ConfigPath: *configPath}); err != nil {
|
||||
return fail(stderr, err)
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func printServeHelp(w io.Writer) {
|
||||
fmt.Fprint(w, `Usage:
|
||||
distributor serve --config <path>
|
||||
|
||||
Options:
|
||||
--config <path> Path to config file
|
||||
|
||||
Serve loads configured HTTP upload sources, resolves upload tokens through the
|
||||
configured secret environment, and starts the HTTP upload API.
|
||||
`)
|
||||
}
|
||||
@@ -1,10 +1,24 @@
|
||||
package config
|
||||
|
||||
type Config struct {
|
||||
Server Server `yaml:"server"`
|
||||
Secrets Secrets `yaml:"secrets"`
|
||||
Pipelines []Pipeline `yaml:"pipelines"`
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
HTTP HTTPServer `yaml:"http"`
|
||||
}
|
||||
|
||||
type HTTPServer struct {
|
||||
Bind string `yaml:"bind"`
|
||||
StagingRoot string `yaml:"staging_root"`
|
||||
MaxUploadSize *ByteSize `yaml:"max_upload_size"`
|
||||
QueueSize int `yaml:"queue_size"`
|
||||
MaxConcurrency int `yaml:"max_concurrency"`
|
||||
Retention *Duration `yaml:"retention"`
|
||||
}
|
||||
|
||||
type Secrets struct {
|
||||
Directory string `yaml:"directory"`
|
||||
}
|
||||
@@ -50,6 +64,13 @@ type Backend struct {
|
||||
ForcePath *bool `yaml:"force_path_style"`
|
||||
Creds Credentials `yaml:"credentials"`
|
||||
SSH SSH `yaml:",inline"`
|
||||
Upload HTTPUpload `yaml:",inline"`
|
||||
}
|
||||
|
||||
type HTTPUpload struct {
|
||||
TokenEnv string `yaml:"token_env"`
|
||||
StagingPath string `yaml:"staging_path"`
|
||||
MaxUploadSize *ByteSize `yaml:"max_upload_size"`
|
||||
}
|
||||
|
||||
type SSH struct {
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
package config
|
||||
|
||||
import "gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
)
|
||||
|
||||
const DefaultConfigPath = "/usr/local/etc/distributor/config.yml"
|
||||
|
||||
const (
|
||||
BackendLocal = "local"
|
||||
BackendSSH = "ssh"
|
||||
BackendS3 = "s3"
|
||||
BackendLocal = "local"
|
||||
BackendSSH = "ssh"
|
||||
BackendS3 = "s3"
|
||||
BackendHTTPUpload = "http_upload"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -38,10 +44,23 @@ const (
|
||||
|
||||
const DefaultS3Region = "us-east-1"
|
||||
|
||||
const (
|
||||
DefaultHTTPBind = "127.0.0.1:8080"
|
||||
DefaultHTTPStagingRoot = "/var/spool/distributor"
|
||||
DefaultHTTPMaxUploadSize = ByteSize(20 * 1024 * 1024)
|
||||
DefaultHTTPQueueSize = 16
|
||||
DefaultHTTPMaxConcurrency = 1
|
||||
DefaultHTTPRetention = Duration(24 * time.Hour)
|
||||
)
|
||||
|
||||
func ApplyDefaults(cfg *Config) {
|
||||
applyHTTPServerDefaults(&cfg.Server.HTTP)
|
||||
for pipelineIndex := range cfg.Pipelines {
|
||||
pipeline := &cfg.Pipelines[pipelineIndex]
|
||||
applyBackendDefaults(&pipeline.Source)
|
||||
if pipeline.Source.Backend == BackendHTTPUpload {
|
||||
applyHTTPUploadDefaults(&pipeline.Source.Upload, pipeline.ID, cfg.Server.HTTP)
|
||||
}
|
||||
if pipeline.Validation.OnDigestMismatch == "" {
|
||||
pipeline.Validation.OnDigestMismatch = ValidationActionFail
|
||||
}
|
||||
@@ -76,6 +95,44 @@ func ApplyDefaults(cfg *Config) {
|
||||
}
|
||||
}
|
||||
|
||||
func applyHTTPServerDefaults(server *HTTPServer) {
|
||||
if server.Bind == "" {
|
||||
server.Bind = DefaultHTTPBind
|
||||
}
|
||||
if server.StagingRoot == "" {
|
||||
server.StagingRoot = DefaultHTTPStagingRoot
|
||||
}
|
||||
if server.MaxUploadSize == nil {
|
||||
server.MaxUploadSize = byteSize(DefaultHTTPMaxUploadSize)
|
||||
}
|
||||
if server.QueueSize == 0 {
|
||||
server.QueueSize = DefaultHTTPQueueSize
|
||||
}
|
||||
if server.MaxConcurrency == 0 {
|
||||
server.MaxConcurrency = DefaultHTTPMaxConcurrency
|
||||
}
|
||||
if server.Retention == nil {
|
||||
server.Retention = duration(DefaultHTTPRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func applyHTTPUploadDefaults(upload *HTTPUpload, pipelineID string, server HTTPServer) {
|
||||
if upload.StagingPath == "" && pipelineID != "" {
|
||||
upload.StagingPath = filepath.Join(server.StagingRoot, pipelineID)
|
||||
}
|
||||
if upload.MaxUploadSize == nil && server.MaxUploadSize != nil {
|
||||
upload.MaxUploadSize = byteSize(*server.MaxUploadSize)
|
||||
}
|
||||
}
|
||||
|
||||
func byteSize(value ByteSize) *ByteSize {
|
||||
return &value
|
||||
}
|
||||
|
||||
func duration(value Duration) *Duration {
|
||||
return &value
|
||||
}
|
||||
|
||||
func applyBackendDefaults(backend *Backend) {
|
||||
if backend.Backend == BackendSSH {
|
||||
if backend.Port == 0 {
|
||||
|
||||
@@ -208,6 +208,123 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsHTTPServerConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`)
|
||||
|
||||
server := cfg.Server.HTTP
|
||||
if got, want := server.Bind, DefaultHTTPBind; got != want {
|
||||
t.Fatalf("server.http.bind = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := server.StagingRoot, DefaultHTTPStagingRoot; got != want {
|
||||
t.Fatalf("server.http.staging_root = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := *server.MaxUploadSize, DefaultHTTPMaxUploadSize; got != want {
|
||||
t.Fatalf("server.http.max_upload_size = %s, want %s", got, want)
|
||||
}
|
||||
if got, want := server.QueueSize, DefaultHTTPQueueSize; got != want {
|
||||
t.Fatalf("server.http.queue_size = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := server.MaxConcurrency, DefaultHTTPMaxConcurrency; got != want {
|
||||
t.Fatalf("server.http.max_concurrency = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := *server.Retention, DefaultHTTPRetention; got != want {
|
||||
t.Fatalf("server.http.retention = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsHTTPUploadSourceConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:9090
|
||||
staging_root: /srv/distributor/staging
|
||||
max_upload_size: 64MB
|
||||
queue_size: 32
|
||||
max_concurrency: 2
|
||||
retention: 48h
|
||||
pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
staging_path: /srv/distributor/staging/weather-daily
|
||||
max_upload_size: 32MB
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`)
|
||||
|
||||
server := cfg.Server.HTTP
|
||||
if got, want := server.Bind, "127.0.0.1:9090"; got != want {
|
||||
t.Fatalf("server.http.bind = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := server.StagingRoot, "/srv/distributor/staging"; got != want {
|
||||
t.Fatalf("server.http.staging_root = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := *server.MaxUploadSize, ByteSize(64*1024*1024); got != want {
|
||||
t.Fatalf("server.http.max_upload_size = %s, want %s", got, want)
|
||||
}
|
||||
if got, want := server.QueueSize, 32; got != want {
|
||||
t.Fatalf("server.http.queue_size = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := server.MaxConcurrency, 2; got != want {
|
||||
t.Fatalf("server.http.max_concurrency = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := server.Retention.String(), "48h0m0s"; got != want {
|
||||
t.Fatalf("server.http.retention = %s, want %s", got, want)
|
||||
}
|
||||
|
||||
source := cfg.Pipelines[0].Source
|
||||
if got, want := source.Backend, BackendHTTPUpload; got != want {
|
||||
t.Fatalf("source.backend = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.Upload.TokenEnv, "WEATHER_DAILY_UPLOAD_TOKEN"; got != want {
|
||||
t.Fatalf("source.token_env = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.Upload.StagingPath, "/srv/distributor/staging/weather-daily"; got != want {
|
||||
t.Fatalf("source.staging_path = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := *source.Upload.MaxUploadSize, ByteSize(32*1024*1024); got != want {
|
||||
t.Fatalf("source.max_upload_size = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsHTTPUploadSourceConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
server:
|
||||
http:
|
||||
max_upload_size: 12MB
|
||||
pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
`)
|
||||
|
||||
source := cfg.Pipelines[0].Source
|
||||
if got, want := source.Upload.StagingPath, "/var/spool/distributor/weather-daily"; got != want {
|
||||
t.Fatalf("source.staging_path = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := *source.Upload.MaxUploadSize, ByteSize(12*1024*1024); got != want {
|
||||
t.Fatalf("source.max_upload_size = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileValidBackendConfigs(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"local": `
|
||||
@@ -396,6 +513,26 @@ func TestLoadFileRejectsInvalidS3Config(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidHTTPUploadConfig(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"server size": `server: {http: {max_upload_size: 20XB}}`,
|
||||
"source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"zero source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"server duration": `server: {http: {retention: forever}}`,
|
||||
"zero server duration": `server: {http: {retention: 0s}}`,
|
||||
"missing token env": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"destination http upload": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: ingest, backend: http_upload}]}]`,
|
||||
"literal token": `pipelines: [{id: reports, source: {backend: http_upload, token: secret, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"unknown server field": `server: {http: {surprise: true}}`,
|
||||
"unknown source field": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assertLoadError(t, body, "")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileDefaultsSSHConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
@@ -581,6 +718,7 @@ func TestExampleConfigsLoad(t *testing.T) {
|
||||
"../../examples/local-index.yml",
|
||||
"../../examples/fan-out.yml",
|
||||
"../../examples/archive-and-latest.yml",
|
||||
"../../examples/http-upload-local.yml",
|
||||
"../../examples/ssh-destination.yml",
|
||||
"../../examples/s3-destination.yml",
|
||||
} {
|
||||
|
||||
117
internal/config/quantity.go
Normal file
117
internal/config/quantity.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type ByteSize int64
|
||||
|
||||
type Duration time.Duration
|
||||
|
||||
func (size *ByteSize) UnmarshalYAML(value *yaml.Node) error {
|
||||
if value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
|
||||
return fmt.Errorf("size must be a string with B, KB, MB, or GB suffix")
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := value.Decode(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
parsed, err := ParseByteSize(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*size = parsed
|
||||
return nil
|
||||
}
|
||||
|
||||
func (size ByteSize) String() string {
|
||||
value := int64(size)
|
||||
if value == 0 {
|
||||
return "0B"
|
||||
}
|
||||
units := []struct {
|
||||
suffix string
|
||||
multiplier int64
|
||||
}{
|
||||
{suffix: "GB", multiplier: 1024 * 1024 * 1024},
|
||||
{suffix: "MB", multiplier: 1024 * 1024},
|
||||
{suffix: "KB", multiplier: 1024},
|
||||
{suffix: "B", multiplier: 1},
|
||||
}
|
||||
for _, unit := range units {
|
||||
if value%unit.multiplier == 0 {
|
||||
return strconv.FormatInt(value/unit.multiplier, 10) + unit.suffix
|
||||
}
|
||||
}
|
||||
return strconv.FormatInt(value, 10) + "B"
|
||||
}
|
||||
|
||||
func ParseByteSize(raw string) (ByteSize, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return 0, fmt.Errorf("size is required")
|
||||
}
|
||||
|
||||
units := []struct {
|
||||
suffix string
|
||||
multiplier int64
|
||||
}{
|
||||
{suffix: "GB", multiplier: 1024 * 1024 * 1024},
|
||||
{suffix: "MB", multiplier: 1024 * 1024},
|
||||
{suffix: "KB", multiplier: 1024},
|
||||
{suffix: "B", multiplier: 1},
|
||||
}
|
||||
for _, unit := range units {
|
||||
number, ok := strings.CutSuffix(value, unit.suffix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(number) != number || number == "" {
|
||||
return 0, fmt.Errorf("size must be an integer followed by B, KB, MB, or GB")
|
||||
}
|
||||
parsed, err := strconv.ParseInt(number, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("size must be an integer followed by B, KB, MB, or GB")
|
||||
}
|
||||
if parsed < 0 {
|
||||
return 0, fmt.Errorf("size must be non-negative")
|
||||
}
|
||||
const maxInt64 = int64(1<<63 - 1)
|
||||
if parsed > 0 && parsed > maxInt64/unit.multiplier {
|
||||
return 0, fmt.Errorf("size is too large")
|
||||
}
|
||||
return ByteSize(parsed * unit.multiplier), nil
|
||||
}
|
||||
return 0, fmt.Errorf("size must use B, KB, MB, or GB suffix")
|
||||
}
|
||||
|
||||
func (duration *Duration) UnmarshalYAML(value *yaml.Node) error {
|
||||
if value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
|
||||
return fmt.Errorf("duration must be a string duration")
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := value.Decode(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
parsed, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("duration must be a valid duration: %w", err)
|
||||
}
|
||||
*duration = Duration(parsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (duration Duration) String() string {
|
||||
return time.Duration(duration).String()
|
||||
}
|
||||
|
||||
func (duration Duration) AsDuration() time.Duration {
|
||||
return time.Duration(duration)
|
||||
}
|
||||
@@ -22,6 +22,8 @@ func (e ValidationErrors) Error() string {
|
||||
func Validate(cfg Config) error {
|
||||
var errs ValidationErrors
|
||||
|
||||
errs = validateHTTPServer(errs, "server.http", cfg.Server.HTTP)
|
||||
|
||||
if len(cfg.Pipelines) == 0 {
|
||||
errs = append(errs, "pipelines is required")
|
||||
}
|
||||
@@ -72,14 +74,56 @@ func Validate(cfg Config) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateHTTPServer(errs ValidationErrors, context string, server HTTPServer) ValidationErrors {
|
||||
if server.Bind == "" {
|
||||
errs = append(errs, context+".bind is required")
|
||||
}
|
||||
if server.StagingRoot == "" {
|
||||
errs = append(errs, context+".staging_root is required")
|
||||
}
|
||||
if server.MaxUploadSize == nil || *server.MaxUploadSize <= 0 {
|
||||
errs = append(errs, context+".max_upload_size must be greater than zero")
|
||||
}
|
||||
if server.QueueSize <= 0 {
|
||||
errs = append(errs, context+".queue_size must be greater than zero")
|
||||
}
|
||||
if server.MaxConcurrency <= 0 {
|
||||
errs = append(errs, context+".max_concurrency must be greater than zero")
|
||||
}
|
||||
if server.Retention == nil || *server.Retention <= 0 {
|
||||
errs = append(errs, context+".retention must be greater than zero")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateSourceBackend(errs ValidationErrors, context string, backend Backend) ValidationErrors {
|
||||
if backend.Backend == BackendHTTPUpload {
|
||||
return validateHTTPUploadSource(errs, context, backend.Upload)
|
||||
}
|
||||
return validateBackend(errs, context, backend.Backend, backend.Host, backend.Port, backend.Path, backend.Endpoint, backend.Bucket, backend.Prefix, backend.SSH.HostKeyPolicy, backend.Creds)
|
||||
}
|
||||
|
||||
func validateDestinationBackend(errs ValidationErrors, context string, destination Destination) ValidationErrors {
|
||||
if destination.Backend == BackendHTTPUpload {
|
||||
errs = append(errs, context+".backend "+BackendHTTPUpload+" is only supported for sources")
|
||||
return errs
|
||||
}
|
||||
return validateBackend(errs, context, destination.Backend, destination.Host, destination.Port, destination.Path, destination.Endpoint, destination.Bucket, destination.Prefix, destination.SSH.HostKeyPolicy, destination.Creds)
|
||||
}
|
||||
|
||||
func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTPUpload) ValidationErrors {
|
||||
if upload.TokenEnv == "" {
|
||||
errs = append(errs, context+".token_env is required for http_upload backend")
|
||||
}
|
||||
if upload.StagingPath == "" {
|
||||
errs = append(errs, context+".staging_path is required for http_upload backend")
|
||||
}
|
||||
if upload.MaxUploadSize == nil || *upload.MaxUploadSize <= 0 {
|
||||
errs = append(errs, context+".max_upload_size must be greater than zero")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateBackend(errs ValidationErrors, context, backend, host string, port int, path, endpoint, bucket, prefix string, hostKeyPolicy HostKeyPolicy, creds Credentials) ValidationErrors {
|
||||
switch backend {
|
||||
case "":
|
||||
|
||||
346
internal/ingest/archive.go
Normal file
346
internal/ingest/archive.go
Normal file
@@ -0,0 +1,346 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
)
|
||||
|
||||
const (
|
||||
ContentTypeTar = "application/x-tar"
|
||||
ContentTypeGzip = "application/gzip"
|
||||
ContentTypeXGzip = "application/x-gzip"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnsupportedContentType = errors.New("unsupported archive content type")
|
||||
ErrUploadTooLarge = errors.New("upload exceeds maximum size")
|
||||
ErrExtractedTooLarge = errors.New("extracted bundle exceeds maximum size")
|
||||
ErrTooManyFiles = errors.New("extracted bundle has too many files")
|
||||
ErrUnsafeArchivePath = errors.New("unsafe archive path")
|
||||
)
|
||||
|
||||
type StageOptions struct {
|
||||
Body io.Reader
|
||||
ContentType string
|
||||
PipelineStagingPath string
|
||||
RunID string
|
||||
MaxUploadSize int64
|
||||
MaxExtractedSize int64
|
||||
MaxFileCount int
|
||||
}
|
||||
|
||||
type StagedBundle struct {
|
||||
Root string
|
||||
Manifest sourcebundle.Manifest
|
||||
}
|
||||
|
||||
func StageArchive(ctx context.Context, opts StageOptions) (StagedBundle, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := validateStageOptions(opts); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
format, err := archiveFormat(opts.ContentType)
|
||||
if err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(opts.PipelineStagingPath, 0o755); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create pipeline staging path: %w", err)
|
||||
}
|
||||
tempDir, err := os.MkdirTemp(opts.PipelineStagingPath, "."+opts.RunID+"-")
|
||||
if err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create staging temp dir: %w", err)
|
||||
}
|
||||
cleanupTemp := true
|
||||
defer func() {
|
||||
if cleanupTemp {
|
||||
_ = os.RemoveAll(tempDir)
|
||||
}
|
||||
}()
|
||||
|
||||
archivePath := filepath.Join(tempDir, "upload.archive")
|
||||
if err := writeLimited(ctx, archivePath, opts.Body, opts.MaxUploadSize); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
extractRoot := filepath.Join(tempDir, "bundle")
|
||||
if err := os.Mkdir(extractRoot, 0o755); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("create extraction root: %w", err)
|
||||
}
|
||||
if err := extractArchive(ctx, archivePath, extractRoot, format, opts.MaxExtractedSize, opts.MaxFileCount); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
manifest, err := sourcebundle.LoadManifest(extractRoot)
|
||||
if err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
if err := sourcebundle.ValidateBundle(extractRoot, manifest); err != nil {
|
||||
return StagedBundle{}, err
|
||||
}
|
||||
|
||||
finalRoot := filepath.Join(opts.PipelineStagingPath, opts.RunID)
|
||||
if err := os.Rename(extractRoot, finalRoot); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("commit staged bundle: %w", err)
|
||||
}
|
||||
cleanupTemp = false
|
||||
if err := os.RemoveAll(tempDir); err != nil {
|
||||
return StagedBundle{}, fmt.Errorf("remove staging temp dir: %w", err)
|
||||
}
|
||||
return StagedBundle{
|
||||
Root: finalRoot,
|
||||
Manifest: manifest,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateStageOptions(opts StageOptions) error {
|
||||
if opts.Body == nil {
|
||||
return fmt.Errorf("body is required")
|
||||
}
|
||||
if opts.PipelineStagingPath == "" {
|
||||
return fmt.Errorf("pipeline staging path is required")
|
||||
}
|
||||
if err := validateRunID(opts.RunID); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.MaxUploadSize <= 0 {
|
||||
return fmt.Errorf("max upload size must be greater than zero")
|
||||
}
|
||||
if opts.MaxExtractedSize <= 0 {
|
||||
return fmt.Errorf("max extracted size must be greater than zero")
|
||||
}
|
||||
if opts.MaxFileCount <= 0 {
|
||||
return fmt.Errorf("max file count must be greater than zero")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRunID(value string) error {
|
||||
if value == "" {
|
||||
return fmt.Errorf("run id is required")
|
||||
}
|
||||
if value == "." || value == ".." || strings.ContainsAny(value, `/\`) {
|
||||
return fmt.Errorf("run id must be a single filesystem path segment")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type archiveKind int
|
||||
|
||||
const (
|
||||
archiveKindTar archiveKind = iota + 1
|
||||
archiveKindGzip
|
||||
)
|
||||
|
||||
func archiveFormat(contentType string) (archiveKind, error) {
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil {
|
||||
mediaType = contentType
|
||||
}
|
||||
switch mediaType {
|
||||
case ContentTypeTar:
|
||||
return archiveKindTar, nil
|
||||
case ContentTypeGzip, ContentTypeXGzip:
|
||||
return archiveKindGzip, nil
|
||||
default:
|
||||
return 0, ErrUnsupportedContentType
|
||||
}
|
||||
}
|
||||
|
||||
func writeLimited(ctx context.Context, destination string, body io.Reader, maxSize int64) error {
|
||||
file, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create upload archive: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
limited := &limitedReader{ctx: ctx, reader: body, limit: maxSize}
|
||||
if _, err := io.Copy(file, limited); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("write upload archive: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type limitedReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
limit int64
|
||||
read int64
|
||||
}
|
||||
|
||||
func (r *limitedReader) Read(data []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if r.read == r.limit {
|
||||
var probe [1]byte
|
||||
n, err := r.reader.Read(probe[:])
|
||||
if n > 0 {
|
||||
return 0, ErrUploadTooLarge
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
remaining := r.limit - r.read
|
||||
if int64(len(data)) > remaining+1 {
|
||||
data = data[:remaining+1]
|
||||
}
|
||||
n, err := r.reader.Read(data)
|
||||
if r.read+int64(n) > r.limit {
|
||||
allowed := int(r.limit - r.read)
|
||||
r.read = r.limit
|
||||
return allowed, ErrUploadTooLarge
|
||||
}
|
||||
r.read += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func extractArchive(ctx context.Context, archivePath, destination string, format archiveKind, maxExtractedSize int64, maxFileCount int) error {
|
||||
file, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open upload archive: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var reader io.Reader = file
|
||||
var gzipReader *gzip.Reader
|
||||
if format == archiveKindGzip {
|
||||
gzipReader, err = gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open gzip archive: %w", err)
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
reader = gzipReader
|
||||
}
|
||||
|
||||
extractor := archiveExtractor{
|
||||
ctx: ctx,
|
||||
destination: destination,
|
||||
maxExtractedSize: maxExtractedSize,
|
||||
maxFileCount: maxFileCount,
|
||||
}
|
||||
if err := extractor.extract(tar.NewReader(reader)); err != nil {
|
||||
return err
|
||||
}
|
||||
if extractor.rootManifestCount != 1 {
|
||||
return fmt.Errorf("archive must contain exactly one root-level manifest.json")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type archiveExtractor struct {
|
||||
ctx context.Context
|
||||
destination string
|
||||
maxExtractedSize int64
|
||||
maxFileCount int
|
||||
extractedSize int64
|
||||
fileCount int
|
||||
rootManifestCount int
|
||||
seenFiles map[string]struct{}
|
||||
}
|
||||
|
||||
func (e *archiveExtractor) extract(reader *tar.Reader) error {
|
||||
e.seenFiles = make(map[string]struct{})
|
||||
for {
|
||||
if err := e.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read tar archive: %w", err)
|
||||
}
|
||||
name, err := cleanArchivePath(header.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if path.Base(name) == sourcebundle.ManifestName {
|
||||
if name != sourcebundle.ManifestName {
|
||||
return fmt.Errorf("nested manifest %q is not allowed", name)
|
||||
}
|
||||
e.rootManifestCount++
|
||||
}
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(filepath.Join(e.destination, filepath.FromSlash(name)), 0o755); err != nil {
|
||||
return fmt.Errorf("create archive directory %q: %w", name, err)
|
||||
}
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
if err := e.extractFile(reader, name, header.Size); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("archive entry %q has unsupported type %c", name, header.Typeflag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *archiveExtractor) extractFile(reader *tar.Reader, name string, size int64) error {
|
||||
if size < 0 {
|
||||
return fmt.Errorf("archive entry %q has invalid size", name)
|
||||
}
|
||||
e.fileCount++
|
||||
if e.fileCount > e.maxFileCount {
|
||||
return ErrTooManyFiles
|
||||
}
|
||||
e.extractedSize += size
|
||||
if e.extractedSize > e.maxExtractedSize {
|
||||
return ErrExtractedTooLarge
|
||||
}
|
||||
if _, exists := e.seenFiles[name]; exists {
|
||||
return fmt.Errorf("archive entry %q is duplicated", name)
|
||||
}
|
||||
e.seenFiles[name] = struct{}{}
|
||||
|
||||
fullPath := filepath.Join(e.destination, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create archive parent for %q: %w", name, err)
|
||||
}
|
||||
file, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create archive file %q: %w", name, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := io.CopyN(file, reader, size); err != nil {
|
||||
return fmt.Errorf("extract archive file %q: %w", name, err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return fmt.Errorf("extract archive file %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanArchivePath(value string) (string, error) {
|
||||
if value == "" || strings.Contains(value, `\`) || strings.HasPrefix(value, "/") {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
cleaned := path.Clean(value)
|
||||
if cleaned != value {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
for _, segment := range strings.Split(value, "/") {
|
||||
if segment == "" || segment == "." || segment == ".." {
|
||||
return "", ErrUnsafeArchivePath
|
||||
}
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
382
internal/ingest/archive_test.go
Normal file
382
internal/ingest/archive_test.go
Normal file
@@ -0,0 +1,382 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sourcebundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
)
|
||||
|
||||
func TestStageArchiveAcceptsTar(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
staged := stageArchive(t, archive, ContentTypeTar)
|
||||
|
||||
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
|
||||
t.Fatalf("manifest id = %q, want %q", got, want)
|
||||
}
|
||||
if got := readFile(t, staged.Root, "report.md"); got != "# Report\n" {
|
||||
t.Fatalf("report = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveAcceptsGzipTar(t *testing.T) {
|
||||
archive := validArchive(t, true)
|
||||
staged := stageArchive(t, archive, ContentTypeGzip+"; charset=binary")
|
||||
|
||||
if got, want := staged.Manifest.ID, "reports.ingest"; got != want {
|
||||
t.Fatalf("manifest id = %q, want %q", got, want)
|
||||
}
|
||||
if got := readFile(t, staged.Root, "summary.txt"); got != "Summary\n" {
|
||||
t.Fatalf("summary = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsUnsupportedContentType(t *testing.T) {
|
||||
err := stageArchiveError(t, validArchive(t, false), "application/zip", nil)
|
||||
if !errors.Is(err, ErrUnsupportedContentType) {
|
||||
t.Fatalf("StageArchive() error = %v, want ErrUnsupportedContentType", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveEnforcesMaxUploadSize(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
err := stageArchiveError(t, archive, ContentTypeTar, func(opts *StageOptions) {
|
||||
opts.MaxUploadSize = int64(len(archive) - 1)
|
||||
})
|
||||
if !errors.Is(err, ErrUploadTooLarge) {
|
||||
t.Fatalf("StageArchive() error = %v, want ErrUploadTooLarge", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveEnforcesExtractionLimits(t *testing.T) {
|
||||
archive := validArchive(t, false)
|
||||
tests := map[string]struct {
|
||||
mutate func(*StageOptions)
|
||||
wantErr error
|
||||
}{
|
||||
"size": {
|
||||
mutate: func(opts *StageOptions) {
|
||||
opts.MaxExtractedSize = 1
|
||||
},
|
||||
wantErr: ErrExtractedTooLarge,
|
||||
},
|
||||
"files": {
|
||||
mutate: func(opts *StageOptions) {
|
||||
opts.MaxFileCount = 1
|
||||
},
|
||||
wantErr: ErrTooManyFiles,
|
||||
},
|
||||
}
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, archive, ContentTypeTar, tt.mutate)
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("StageArchive() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsUnsafeEntries(t *testing.T) {
|
||||
tests := map[string][]tarEntry{
|
||||
"absolute path": {
|
||||
fileEntry("/report.md", "report"),
|
||||
},
|
||||
"path traversal": {
|
||||
fileEntry("../report.md", "report"),
|
||||
},
|
||||
"backslash path": {
|
||||
fileEntry(`nested\report.md`, "report"),
|
||||
},
|
||||
"symlink": {
|
||||
{name: "link.md", typeflag: tar.TypeSymlink, linkname: "report.md"},
|
||||
},
|
||||
"hardlink": {
|
||||
{name: "link.md", typeflag: tar.TypeLink, linkname: "report.md"},
|
||||
},
|
||||
"device": {
|
||||
{name: "device", typeflag: tar.TypeChar},
|
||||
},
|
||||
}
|
||||
for name, entries := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveRejectsBundleValidationFailures(t *testing.T) {
|
||||
tests := map[string][]tarEntry{
|
||||
"missing manifest": {
|
||||
fileEntry("report.md", "report"),
|
||||
},
|
||||
"nested manifest": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.nested", fileSpec{path: "report.md", body: "report"}))),
|
||||
fileEntry("nested/manifest.json", "{}"),
|
||||
fileEntry("report.md", "report"),
|
||||
},
|
||||
"missing listed file": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.missing", fileSpec{path: "missing.md", body: "missing"}))),
|
||||
},
|
||||
"digest mismatch": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.digest", fileSpec{path: "report.md", body: "expected"}))),
|
||||
fileEntry("report.md", "actual"),
|
||||
},
|
||||
"non regular listed file": {
|
||||
fileEntry("manifest.json", manifestJSON(t, manifestFor("reports.directory", fileSpec{path: "report.md", body: "report"}))),
|
||||
{name: "report.md", typeflag: tar.TypeDir},
|
||||
},
|
||||
}
|
||||
for name, entries := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := stageArchiveError(t, makeArchive(t, false, entries...), ContentTypeTar, nil)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageArchiveCleansUpFailedExtraction(t *testing.T) {
|
||||
stagingPath := filepath.Join(t.TempDir(), "staging")
|
||||
archive := makeArchive(t, false, fileEntry("../report.md", "report"))
|
||||
_, err := StageArchive(context.Background(), StageOptions{
|
||||
Body: bytes.NewReader(archive),
|
||||
ContentType: ContentTypeTar,
|
||||
PipelineStagingPath: stagingPath,
|
||||
RunID: "reports.20260603T120000Z.abcd",
|
||||
MaxUploadSize: int64(len(archive)),
|
||||
MaxExtractedSize: 1024 * 1024,
|
||||
MaxFileCount: 10,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
entries, err := os.ReadDir(stagingPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir() error = %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("staging entries = %d, want cleanup", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func stageArchive(t *testing.T, archive []byte, contentType string) StagedBundle {
|
||||
t.Helper()
|
||||
staged, err := StageArchive(context.Background(), defaultStageOptions(t, archive, contentType))
|
||||
if err != nil {
|
||||
t.Fatalf("StageArchive() error = %v", err)
|
||||
}
|
||||
return staged
|
||||
}
|
||||
|
||||
func stageArchiveError(t *testing.T, archive []byte, contentType string, mutate func(*StageOptions)) error {
|
||||
t.Helper()
|
||||
opts := defaultStageOptions(t, archive, contentType)
|
||||
if mutate != nil {
|
||||
mutate(&opts)
|
||||
}
|
||||
_, err := StageArchive(context.Background(), opts)
|
||||
if err == nil {
|
||||
t.Fatal("StageArchive() error = nil, want error")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func defaultStageOptions(t *testing.T, archive []byte, contentType string) StageOptions {
|
||||
t.Helper()
|
||||
return StageOptions{
|
||||
Body: bytes.NewReader(archive),
|
||||
ContentType: contentType,
|
||||
PipelineStagingPath: filepath.Join(t.TempDir(), "staging"),
|
||||
RunID: "reports.20260603T120000Z.abcd",
|
||||
MaxUploadSize: int64(len(archive)),
|
||||
MaxExtractedSize: 1024 * 1024,
|
||||
MaxFileCount: 10,
|
||||
}
|
||||
}
|
||||
|
||||
func validArchive(t *testing.T, compressed bool) []byte {
|
||||
t.Helper()
|
||||
root := filepath.Join(t.TempDir(), "bundle")
|
||||
sourceRoot := t.TempDir()
|
||||
writeFile(t, sourceRoot, "report.md", "# Report\n")
|
||||
writeFile(t, sourceRoot, "summary.txt", "Summary\n")
|
||||
_, err := sourcebundle.WriteBundle(sourcebundle.WriteBundleOptions{
|
||||
Root: root,
|
||||
ID: "reports.ingest",
|
||||
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
|
||||
Files: []sourcebundle.BundleFile{
|
||||
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||
{SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "summary.txt"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteBundle() error = %v", err)
|
||||
}
|
||||
|
||||
var entries []tarEntry
|
||||
if err := filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
relative, err := filepath.Rel(root, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries = append(entries, fileEntry(filepath.ToSlash(relative), string(data)))
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("walk bundle: %v", err)
|
||||
}
|
||||
return makeArchive(t, compressed, entries...)
|
||||
}
|
||||
|
||||
type tarEntry struct {
|
||||
name string
|
||||
typeflag byte
|
||||
body []byte
|
||||
linkname string
|
||||
}
|
||||
|
||||
func fileEntry(name, body string) tarEntry {
|
||||
return tarEntry{name: name, typeflag: tar.TypeReg, body: []byte(body)}
|
||||
}
|
||||
|
||||
func makeArchive(t *testing.T, compressed bool, entries ...tarEntry) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
var writer *tar.Writer
|
||||
var gzipWriter *gzip.Writer
|
||||
if compressed {
|
||||
gzipWriter = gzip.NewWriter(&output)
|
||||
writer = tar.NewWriter(gzipWriter)
|
||||
} else {
|
||||
writer = tar.NewWriter(&output)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
header := &tar.Header{
|
||||
Name: entry.name,
|
||||
Typeflag: entry.typeflag,
|
||||
Size: int64(len(entry.body)),
|
||||
Mode: 0o644,
|
||||
Linkname: entry.linkname,
|
||||
}
|
||||
if entry.typeflag == tar.TypeDir {
|
||||
header.Size = 0
|
||||
header.Mode = 0o755
|
||||
}
|
||||
if err := writer.WriteHeader(header); err != nil {
|
||||
t.Fatalf("WriteHeader(%q) error = %v", entry.name, err)
|
||||
}
|
||||
if len(entry.body) > 0 {
|
||||
if _, err := writer.Write(entry.body); err != nil {
|
||||
t.Fatalf("Write(%q) error = %v", entry.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close tar writer: %v", err)
|
||||
}
|
||||
if gzipWriter != nil {
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
t.Fatalf("close gzip writer: %v", err)
|
||||
}
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
type fileSpec struct {
|
||||
path string
|
||||
body string
|
||||
}
|
||||
|
||||
func manifestFor(id string, files ...fileSpec) sourcebundle.Manifest {
|
||||
manifest := sourcebundle.Manifest{
|
||||
SchemaVersion: sourcebundle.SchemaVersion,
|
||||
ID: id,
|
||||
Created: time.Date(2026, 6, 3, 12, 0, 0, 0, time.UTC),
|
||||
}
|
||||
for _, file := range files {
|
||||
manifest.Files = append(manifest.Files, sourcebundle.ManifestFile{
|
||||
Path: file.path,
|
||||
SHA256: sourcebundle.FileDigest([]byte(file.body)),
|
||||
Size: int64(len(file.body)),
|
||||
})
|
||||
}
|
||||
manifest.Digest = sourcebundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
}
|
||||
|
||||
func manifestJSON(t *testing.T, manifest sourcebundle.Manifest) string {
|
||||
t.Helper()
|
||||
data, err := sourcebundle.MarshalManifest(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalManifest() error = %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, root, relative, body string) {
|
||||
t.Helper()
|
||||
fullPath := filepath.Join(root, filepath.FromSlash(relative))
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(fullPath, []byte(body), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, root, relative string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative)))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func TestCleanArchivePath(t *testing.T) {
|
||||
tests := map[string]bool{
|
||||
"manifest.json": true,
|
||||
"nested/report.md": true,
|
||||
"": false,
|
||||
"/absolute.md": false,
|
||||
"../escape.md": false,
|
||||
"nested/../report.md": false,
|
||||
`nested\report.md`: false,
|
||||
"./report.md": false,
|
||||
"nested//report.md": false,
|
||||
}
|
||||
for value, wantOK := range tests {
|
||||
t.Run(strings.ReplaceAll(value, "/", "_"), func(t *testing.T) {
|
||||
_, err := cleanArchivePath(value)
|
||||
if wantOK && err != nil {
|
||||
t.Fatalf("cleanArchivePath(%q) error = %v", value, err)
|
||||
}
|
||||
if !wantOK && err == nil {
|
||||
t.Fatalf("cleanArchivePath(%q) error = nil, want error", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user