Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9aaa1e9426 | |||
| bb276101d2 | |||
| 67dc07435d | |||
| 9684ffd37f | |||
| eba4d6dd56 | |||
| d5e3aa7a44 | |||
| c372a02357 | |||
| 07f1eb2148 | |||
| 4b6a0a3b74 | |||
| 42fb4aa82a | |||
| c4cfd3fc74 | |||
| 004283fc0a | |||
| 063a13b1c9 | |||
| 1fc282f796 | |||
| c8b22d13a2 | |||
| f2f3bdf784 | |||
| 980ae15249 | |||
| a8564035d3 | |||
| 1a52fdce6f | |||
| 29fd0e494c | |||
| 8b1e5abf68 | |||
| 04557f610d | |||
| bb68cb6602 | |||
| e51bc28b05 | |||
| 0382978af0 | |||
| a6c38d3e96 |
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
`distributor` validates manifested report bundles and publishes selected source or generated artifacts to configured destinations.
|
`distributor` validates manifested report bundles and publishes selected source or generated artifacts to configured destinations.
|
||||||
|
|
||||||
It is a local-first CLI with SSH/SFTP and S3-compatible storage support: source bundles can be read from local or remote storage, destinations can be local directories or remote paths, and Markdown files can be rendered to HTML sidecars.
|
It is a local-first CLI with SSH/SFTP and S3-compatible storage support: source bundles can be read from local or remote storage, destinations can be local directories or remote paths, and Markdown files can be rendered to HTML sidecars or `index.html`.
|
||||||
|
|
||||||
|
Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build, write, parse, and validate complete local source bundles with the same manifest contract used by `distributor`.
|
||||||
|
|
||||||
Run the local example pipeline:
|
Run the local example pipeline:
|
||||||
|
|
||||||
@@ -10,4 +12,4 @@ Run the local example pipeline:
|
|||||||
go run ./cmd/distributor run --config examples/local-publish.yml
|
go run ./cmd/distributor run --config examples/local-publish.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
See [docs/cli.md](docs/cli.md), [docs/config.md](docs/config.md), [docs/operations.md](docs/operations.md), and [docs/troubleshooting.md](docs/troubleshooting.md) for the implemented CLI, configuration, operating notes, and common failure modes. Planning material lives under `docs/roadmap/`.
|
See [docs/cli.md](docs/cli.md), [docs/config.md](docs/config.md), [docs/operations.md](docs/operations.md), and [docs/troubleshooting.md](docs/troubleshooting.md) for the implemented CLI, configuration, operating notes, and common failure modes. Future and deferred work lives under `docs/roadmap/`.
|
||||||
|
|||||||
108
docs/cli.md
108
docs/cli.md
@@ -12,18 +12,22 @@ This discovers the example source bundle and publishes source files to `workspac
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
distributor [--help]
|
distributor [--help]
|
||||||
distributor version
|
distributor version [--format text|json]
|
||||||
distributor run [--config <path>] [--dry-run] [--force]
|
distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
|
||||||
distributor validate <path>
|
distributor validate [--format text|json] <path>
|
||||||
distributor inspect <path>
|
distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
|
||||||
|
distributor inspect [--format text|json] <path>
|
||||||
|
distributor inspect --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
|
||||||
|
distributor manifest create <bundle-path> --id <bundle-id> [options]
|
||||||
```
|
```
|
||||||
|
|
||||||
- `version`: prints the application name and version. Development builds print `distributor dev`.
|
- `version`: prints the application name and version. Development builds print `distributor dev`.
|
||||||
- `run`: loads a YAML config, discovers source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary.
|
- `run`: loads a YAML config, discovers source bundles, plans each configured destination, writes selected outputs unless `--dry-run` is set, and prints a final status summary.
|
||||||
- `validate`: validates a local source bundle directory or a local tree containing source bundles.
|
- `validate`: validates a local source bundle directory, a local source bundle tree, or one configured pipeline source.
|
||||||
- `inspect`: validates local source bundles and prints normalized bundle metadata.
|
- `inspect`: validates source bundles and prints normalized bundle metadata for a local path or one configured pipeline source.
|
||||||
|
- `manifest create`: creates `manifest.json` for a local source bundle directory.
|
||||||
|
|
||||||
`validate` and `inspect` accept local paths only. `run` executes `local`, `ssh`, and `s3` backends.
|
`validate` and `inspect` have two mutually exclusive modes: a local path shortcut, or configured source mode with `--config <path> --pipeline <id>`. Configured source mode opens only the selected pipeline source and supports configured `local`, `ssh`, and `s3` sources. It does not open destinations. `run` executes configured sources and destinations.
|
||||||
|
|
||||||
## Flag reference
|
## Flag reference
|
||||||
|
|
||||||
@@ -35,13 +39,30 @@ All subcommands:
|
|||||||
|
|
||||||
- `--help`, `-h`: print command-specific help.
|
- `--help`, `-h`: print command-specific help.
|
||||||
|
|
||||||
|
Output-producing subcommands:
|
||||||
|
|
||||||
|
- `--format text|json`: output format. `text` is the default. Help and usage output are always text.
|
||||||
|
|
||||||
`run` flags:
|
`run` flags:
|
||||||
|
|
||||||
- `--config <path>`: config file to load. If omitted, `run` uses `/usr/local/etc/distributor/config.yml`.
|
- `--config <path>`: config file to load. If omitted, `run` uses `/usr/local/etc/distributor/config.yml`.
|
||||||
- `--dry-run`: load config, discover bundles, inspect destination state, print planned actions and final status, and do not write output files, destination state, or SSH `known_hosts` entries.
|
- `--dry-run`: load config, discover bundles, inspect destination state, print planned actions and final status, and do not write output files, destination state, or SSH `known_hosts` entries.
|
||||||
- `--force`: allow explicit destructive replacement for supported conflict cases in this run only.
|
- `--force`: allow explicit destructive replacement for supported conflict cases in this run only.
|
||||||
|
|
||||||
`run` does not accept positional arguments. `validate` and `inspect` accept at most one path; omitting the path returns a required-path error.
|
`validate` and `inspect` configured source flags:
|
||||||
|
|
||||||
|
- `--config <path>`: config file to load for source validation or inspection. Required in configured source mode.
|
||||||
|
- `--pipeline <id>`: pipeline source to validate or inspect. Required in configured source mode.
|
||||||
|
- `--bundle <path>`: source-root-relative bundle directory to validate or inspect instead of discovering every bundle under the source root.
|
||||||
|
|
||||||
|
`manifest create` flags:
|
||||||
|
|
||||||
|
- `--id <bundle-id>`: source bundle id. Required.
|
||||||
|
- `--file <path>`: bundle-relative file to include. Repeatable. If omitted, files are scanned recursively.
|
||||||
|
- `--created <time>`: RFC3339 source created timestamp. If omitted, the current UTC time is used.
|
||||||
|
- `--overwrite`: replace an existing `manifest.json`.
|
||||||
|
|
||||||
|
`run` does not accept positional arguments. `validate` and `inspect` accept at most one path in local mode. Local paths cannot be combined with `--config`, `--pipeline`, or `--bundle`.
|
||||||
|
|
||||||
## Common workflows
|
## Common workflows
|
||||||
|
|
||||||
@@ -57,6 +78,37 @@ Inspect a source bundle:
|
|||||||
go run ./cmd/distributor inspect examples/source-bundle
|
go run ./cmd/distributor inspect examples/source-bundle
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Validate a configured source without opening destinations:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor validate --config examples/local-publish.yml --pipeline example-source-bundle
|
||||||
|
```
|
||||||
|
|
||||||
|
Inspect one configured source bundle:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor inspect \
|
||||||
|
--config <config-path> \
|
||||||
|
--pipeline <pipeline-id> \
|
||||||
|
--bundle daily/2026-06-01
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a manifest for a local producer bundle:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor manifest create <bundle-path> --id <bundle-id>
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a manifest with explicit file order:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor manifest create <bundle-path> \
|
||||||
|
--id <bundle-id> \
|
||||||
|
--created 2026-06-01T11:00:00Z \
|
||||||
|
--file report.md \
|
||||||
|
--file summary.txt
|
||||||
|
```
|
||||||
|
|
||||||
Preview local publication without writing:
|
Preview local publication without writing:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -81,6 +133,12 @@ Preview local fan-out publication:
|
|||||||
go run ./cmd/distributor run --config examples/fan-out.yml --dry-run
|
go run ./cmd/distributor run --config examples/fan-out.yml --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Preview local archive-plus-latest publication:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/archive-and-latest.yml --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
Preview a forced replacement before publishing:
|
Preview a forced replacement before publishing:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -89,7 +147,9 @@ go run ./cmd/distributor run --config <config-path> --dry-run --force
|
|||||||
|
|
||||||
## Output
|
## Output
|
||||||
|
|
||||||
`run` prints the number of configured pipelines, one line per pipeline, one line per planned destination action, and a final status line. Destination action lines include the bundle path, destination id, destination backend, action, outputs, and reason. Actions include:
|
Text output is the default and is intended for humans.
|
||||||
|
|
||||||
|
`run` text output prints the number of configured pipelines, one line per pipeline, one line per planned destination action, and a final status line. Destination action lines include the source bundle path, destination id, destination backend, action, outputs, and reason. Fixed path destinations also print `path_mapping=fixed target=.` to show that the selected bundle targets the destination backend root. Actions include:
|
||||||
|
|
||||||
- `publish_new`: destination has no managed state and is empty.
|
- `publish_new`: destination has no managed state and is empty.
|
||||||
- `replace_older`: destination state is older than the source manifest.
|
- `replace_older`: destination state is older than the source manifest.
|
||||||
@@ -100,10 +160,36 @@ go run ./cmd/distributor run --config <config-path> --dry-run --force
|
|||||||
|
|
||||||
The command exits non-zero if any destination fails. Independent later destinations are still attempted.
|
The command exits non-zero if any destination fails. Independent later destinations are still attempted.
|
||||||
|
|
||||||
The final status line includes counters for `publish_new`, `replace_older`, `force_replace`, skipped destinations, failures, and whether the run was a dry run.
|
Dry-run output for fixed path destinations prints a warning with the candidate count and selected source bundle. If a fixed path dry run plans a destructive replacement, it prints an additional warning that the destination root would be replaced.
|
||||||
|
|
||||||
|
The final status line includes counters for `publish_new`, `replace_older`, `force_replace`, skipped destinations, failures, whether the run was a dry run, and fixed path destinations.
|
||||||
|
|
||||||
|
JSON output writes exactly one JSON document to stdout:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"command": "inspect",
|
||||||
|
"ok": true,
|
||||||
|
"warnings": [],
|
||||||
|
"result": {}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Warnings are objects in the top-level `warnings` array and are not printed again as text. Fatal setup errors, such as a missing config file or invalid arguments, write no JSON document and return a non-zero exit code with a text error on stderr.
|
||||||
|
|
||||||
|
`run --format json` returns partial results when destination failures occur after planning or execution begins. In that case stdout contains `ok: false`, a `result` with pipeline summaries, destination actions, final counters, and a top-level `errors` array; the process still exits non-zero.
|
||||||
|
|
||||||
|
Command-specific JSON results:
|
||||||
|
|
||||||
|
- `version`: application name and version.
|
||||||
|
- `validate`: bundle count and discovered bundle identifiers. Configured source results also include pipeline id and source backend.
|
||||||
|
- `inspect`: bundle path, id, created timestamp, digest, file count, total size, and manifest file records. Configured source results also include pipeline id and source backend.
|
||||||
|
- `manifest create`: manifest path, bundle root, id, created timestamp, digest, file count, and file records.
|
||||||
|
- `run`: dry-run status, pipeline summaries, destination action records, destination bundle paths, path mapping markers, optional primary URLs, output records with optional URLs, final counters, warnings, and partial failure records.
|
||||||
|
|
||||||
## Diagnostics
|
## Diagnostics
|
||||||
|
|
||||||
Use `validate` before publication when a producer has written a new bundle. Use `inspect` to confirm normalized ids, timestamps, digests, file paths, and file sizes.
|
Use `manifest create` when a local producer has written bundle files but not `manifest.json`. Use `validate` before publication when a producer has written a new bundle; use configured source mode when the bundle is already on an SSH or S3 source. Use `inspect` to confirm normalized ids, timestamps, digests, file paths, and file sizes.
|
||||||
|
|
||||||
For symptom-oriented recovery steps, see [troubleshooting](troubleshooting.md). For destination state and retry behavior, see [operations](operations.md). For config fields and defaults, see [configuration](config.md).
|
For symptom-oriented recovery steps, see [troubleshooting](troubleshooting.md). For destination state and retry behavior, see [operations](operations.md). For config fields and defaults, see [configuration](config.md).
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ pipelines:
|
|||||||
|
|
||||||
## HTML Publication
|
## HTML Publication
|
||||||
|
|
||||||
To publish generated HTML from Markdown files:
|
To publish generated sidecar HTML from Markdown files:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
publish:
|
publish:
|
||||||
@@ -68,6 +68,77 @@ transform:
|
|||||||
|
|
||||||
Sidecar generation writes `report.html` for `report.md`. It does not mutate the source bundle.
|
Sidecar generation writes `report.html` for `report.md`. It does not mutate the source bundle.
|
||||||
|
|
||||||
|
To publish a single Markdown file as `index.html`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: index
|
||||||
|
input: report.md
|
||||||
|
```
|
||||||
|
|
||||||
|
When `mode: index` omits `input`, the source manifest must list exactly one Markdown file.
|
||||||
|
|
||||||
|
## Destination Path Mapping
|
||||||
|
|
||||||
|
Each destination chooses how source bundle paths map into that destination:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
path_mapping:
|
||||||
|
mode: preserve_relative
|
||||||
|
```
|
||||||
|
|
||||||
|
`preserve_relative` is the default. It publishes each discovered source bundle at the same path relative to the destination backend root. A source bundle at `daily/2026-06-01` publishes below `daily/2026-06-01` for that destination.
|
||||||
|
|
||||||
|
`fixed` publishes one selected source bundle directly at the destination backend root:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
destinations:
|
||||||
|
- id: latest-html
|
||||||
|
backend: local
|
||||||
|
path: /srv/www/reports/latest
|
||||||
|
path_mapping:
|
||||||
|
mode: fixed
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: index
|
||||||
|
input: report.md
|
||||||
|
```
|
||||||
|
|
||||||
|
Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple candidates have the same timestamp, the source-root-relative bundle path in ascending order wins. Older candidates are not planned or written for that destination.
|
||||||
|
|
||||||
|
Fixed mapping is useful for stable latest-style paths. It is more destructive than archive-style publication because successive source bundles target the same destination root. Preview fixed destinations with `run --dry-run`, especially before using `--force`.
|
||||||
|
|
||||||
|
## Destination Links
|
||||||
|
|
||||||
|
Destinations can record public URLs for published outputs:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
links:
|
||||||
|
base_url: https://reports.example.com/archive
|
||||||
|
primary: auto
|
||||||
|
```
|
||||||
|
|
||||||
|
`links.base_url` is an absolute `http` or `https` URL corresponding to the destination backend root. It may include a path prefix, but it must not include a query string or fragment. Distributor does not infer public URLs from backend config.
|
||||||
|
|
||||||
|
`links.primary` selects the top-level primary URL stored in destination state:
|
||||||
|
|
||||||
|
- `auto`: prefer `index.html`, then generated HTML, then source outputs.
|
||||||
|
- `html`: use the first generated HTML output.
|
||||||
|
- `source`: use the first copied source output.
|
||||||
|
|
||||||
|
If a destination has no `links` block, no URL metadata is generated. If a primary policy has no matching output, per-output URLs are still recorded and the top-level primary URL is omitted.
|
||||||
|
|
||||||
|
Output URLs are built from `links.base_url`, the destination bundle path, and the output path using URL path semantics. `index.html` outputs produce directory-style URLs that omit the filename.
|
||||||
|
|
||||||
## Reference
|
## Reference
|
||||||
|
|
||||||
Top level:
|
Top level:
|
||||||
@@ -106,6 +177,9 @@ Destination:
|
|||||||
- Backend fields: same accepted shape as source backends, with destination fields at the destination level.
|
- Backend fields: same accepted shape as source backends, with destination fields at the destination level.
|
||||||
- `publish`: optional; defaults to source-only publication.
|
- `publish`: optional; defaults to source-only publication.
|
||||||
- `transform`: required only for generated HTML publication.
|
- `transform`: required only for generated HTML publication.
|
||||||
|
- `path_mapping.mode`: optional; defaults to `preserve_relative`. Accepted values are `preserve_relative` and `fixed`.
|
||||||
|
- `links.base_url`: optional links block; when present, `base_url` is required and must be an absolute HTTP or HTTPS URL without query string or fragment.
|
||||||
|
- `links.primary`: optional; defaults to `auto`. Accepted values are `auto`, `html`, and `source`.
|
||||||
- `transfer`: optional; defaults described below.
|
- `transfer`: optional; defaults described below.
|
||||||
|
|
||||||
Accepted backend names:
|
Accepted backend names:
|
||||||
@@ -164,7 +238,15 @@ Publish policy:
|
|||||||
- `publish.source`: publish source artifacts.
|
- `publish.source`: publish source artifacts.
|
||||||
- `publish.html`: publish generated HTML artifacts from Markdown source files.
|
- `publish.html`: publish generated HTML artifacts from Markdown source files.
|
||||||
|
|
||||||
At least one output type must be enabled. When `publish.html` is true, `transform.markdown_to_html.enabled` must be `true` and `transform.markdown_to_html.mode` must be `sidecar`.
|
At least one output type must be enabled. When `publish.html` is true, `transform.markdown_to_html.enabled` must be `true`.
|
||||||
|
|
||||||
|
Markdown-to-HTML transform:
|
||||||
|
|
||||||
|
- `transform.markdown_to_html.enabled`: enables Markdown-to-HTML generation for destinations with `publish.html: true`.
|
||||||
|
- `transform.markdown_to_html.mode`: optional; defaults to `sidecar`. Accepted values are `sidecar` and `index`.
|
||||||
|
- `transform.markdown_to_html.input`: optional source manifest path for `index` mode. It must identify a listed Markdown file.
|
||||||
|
|
||||||
|
`sidecar` mode renders each manifest-listed `.md` file to a same-directory `.html` output. `index` mode renders one selected Markdown file to `index.html` at the destination bundle path. Enabled Markdown-to-HTML config is rejected when `publish.html` is false, and `input` is valid only with `mode: index`.
|
||||||
|
|
||||||
Transfer policy:
|
Transfer policy:
|
||||||
|
|
||||||
@@ -184,8 +266,11 @@ Defaults are applied after YAML decoding and before validation:
|
|||||||
- SSH `host_key_policy: accept-new`
|
- SSH `host_key_policy: accept-new`
|
||||||
- S3 `region: us-east-1`
|
- S3 `region: us-east-1`
|
||||||
- S3 `force_path_style: true`
|
- S3 `force_path_style: true`
|
||||||
|
- `transform.markdown_to_html.mode: sidecar` when a Markdown-to-HTML transform block is present and mode is omitted
|
||||||
- `publish.source: true`
|
- `publish.source: true`
|
||||||
- `publish.html: false`
|
- `publish.html: false`
|
||||||
|
- `path_mapping.mode: preserve_relative`
|
||||||
|
- `links.primary: auto` when a `links` block is present and `primary` is omitted
|
||||||
- `transfer.on_destination_same: skip`
|
- `transfer.on_destination_same: skip`
|
||||||
- `transfer.on_destination_older: replace`
|
- `transfer.on_destination_older: replace`
|
||||||
- `transfer.on_destination_newer: skip`
|
- `transfer.on_destination_newer: skip`
|
||||||
@@ -216,6 +301,8 @@ Maintained examples live under [examples](../examples/):
|
|||||||
- `local-to-local.yml`: minimal local config.
|
- `local-to-local.yml`: minimal local config.
|
||||||
- `local-publish.yml`: runnable local source publication.
|
- `local-publish.yml`: runnable local source publication.
|
||||||
- `local-html.yml`: runnable local HTML publication.
|
- `local-html.yml`: runnable local HTML publication.
|
||||||
|
- `local-index.yml`: runnable local `index.html` publication.
|
||||||
- `fan-out.yml`: runnable local fan-out publication to source and HTML destinations.
|
- `fan-out.yml`: runnable local fan-out publication to source and HTML destinations.
|
||||||
|
- `archive-and-latest.yml`: runnable local fan-out publication to an archive destination and a fixed latest destination.
|
||||||
- `ssh-destination.yml`: environment-gated local-to-SSH publication example.
|
- `ssh-destination.yml`: environment-gated local-to-SSH publication example.
|
||||||
- `s3-destination.yml`: environment-gated local-to-S3 publication example.
|
- `s3-destination.yml`: environment-gated local-to-S3 publication example.
|
||||||
|
|||||||
@@ -12,7 +12,12 @@ Rendering uses `github.com/yuin/goldmark`. The exact dependency version is pinne
|
|||||||
|
|
||||||
`internal/transform/markdown.New` constructs the renderer with `goldmark.New()` and no project-specific extensions or renderer options.
|
`internal/transform/markdown.New` constructs the renderer with `goldmark.New()` and no project-specific extensions or renderer options.
|
||||||
|
|
||||||
For each source bundle file ending in `.md`, the transform reads the Markdown source and generates an HTML sidecar in the same logical directory. The output path replaces the `.md` suffix with `.html`, so `report.md` produces `report.html`. Non-Markdown source files produce no Markdown outputs.
|
The transform supports two output modes:
|
||||||
|
|
||||||
|
- `sidecar`: reads each source bundle file ending in `.md` and generates an HTML sidecar in the same logical directory. The output path replaces the `.md` suffix with `.html`, so `report.md` produces `report.html`. Non-Markdown source files produce no Markdown outputs.
|
||||||
|
- `index`: renders one selected Markdown source to `index.html` at the destination bundle path.
|
||||||
|
|
||||||
|
In `index` mode, `transform.markdown_to_html.input` can name the source manifest path to render. If `input` is omitted, the manifest must list exactly one Markdown file. The selected input must be a safe relative source path, must be listed in the source manifest, and must end in `.md`.
|
||||||
|
|
||||||
Raw HTML embedded in Markdown is not passed through by the current renderer behavior. Tests allow Goldmark's disabled-or-escaped raw HTML output forms and reject literal script tags in generated HTML.
|
Raw HTML embedded in Markdown is not passed through by the current renderer behavior. Tests allow Goldmark's disabled-or-escaped raw HTML output forms and reject literal script tags in generated HTML.
|
||||||
|
|
||||||
@@ -42,7 +47,7 @@ Generated outputs record:
|
|||||||
|
|
||||||
Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select outputs, or choose transfer actions. Publish planning decides whether generated HTML is selected for a destination.
|
Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select outputs, or choose transfer actions. Publish planning decides whether generated HTML is selected for a destination.
|
||||||
|
|
||||||
Only sidecar output mode is supported for current behavior.
|
Publish planning chooses the configured mode and input for each destination. Markdown rendering does not inspect destinations, publish files, write `.distributor.json`, or choose transfer actions.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
@@ -52,4 +57,4 @@ Before changing Markdown renderer behavior, inspect and run:
|
|||||||
go test ./internal/transform/markdown
|
go test ./internal/transform/markdown
|
||||||
```
|
```
|
||||||
|
|
||||||
The tests cover sidecar naming, ignored non-Markdown files, raw HTML handling, deterministic output, digest metadata, and size metadata.
|
The tests cover sidecar naming, index input selection, ignored non-Markdown files, raw HTML handling, deterministic output, digest metadata, and size metadata.
|
||||||
|
|||||||
@@ -6,9 +6,9 @@
|
|||||||
|
|
||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
|
|
||||||
`Run` accepts a context, optional config path, dry-run flag, force flag, stdout writer, and optional notifier. It loads YAML config, discovers source bundles for each configured pipeline, plans each destination independently, optionally executes publish plans, writes summary output when stdout is supplied, and returns an aggregated error if any destination fails.
|
`Run` accepts a context, optional config path, dry-run flag, force flag, stdout writer, output format, and optional notifier. It loads YAML config, discovers source bundles for each configured pipeline, plans each destination independently, optionally executes publish plans, writes text or JSON output when stdout is supplied, and returns an aggregated error if any destination fails.
|
||||||
|
|
||||||
`Validate` and `Inspect` accept a local path. `Validate` discovers and validates bundles. `Inspect` writes bundle metadata and manifest file entries to stdout when provided.
|
`Validate` and `Inspect` accept either a local path or one configured pipeline source. `Validate` discovers and validates bundles. `Inspect` writes bundle metadata and manifest file entries to stdout when provided.
|
||||||
|
|
||||||
## Run flow
|
## Run flow
|
||||||
|
|
||||||
@@ -17,17 +17,31 @@ The runner:
|
|||||||
1. loads config from the supplied path or `config.DefaultConfigPath`;
|
1. loads config from the supplied path or `config.DefaultConfigPath`;
|
||||||
2. opens the configured source backend;
|
2. opens the configured source backend;
|
||||||
3. discovers validated bundles from the source root;
|
3. discovers validated bundles from the source root;
|
||||||
4. opens each destination backend independently;
|
4. selects source bundles for each destination according to destination path mapping;
|
||||||
5. builds a publish plan for each bundle and destination;
|
5. opens each destination backend independently;
|
||||||
6. prints plan lines and records summary counters;
|
6. builds publish plans for the selected bundle and destination combinations;
|
||||||
7. executes publish or replacement plans unless dry-run is enabled;
|
7. prints plan lines or JSON action records and records summary counters;
|
||||||
8. invokes the notifier after successful publish or replacement actions.
|
8. executes publish or replacement plans unless dry-run is enabled;
|
||||||
|
9. invokes the notifier after successful publish or replacement actions.
|
||||||
|
|
||||||
Destination failures are collected while later destinations continue to run. Source open and source discovery failures stop the run because there are no valid bundles to fan out.
|
Destination failures are collected while later destinations continue to run. Source open and source discovery failures stop the run because there are no valid bundles to fan out.
|
||||||
|
|
||||||
|
## Run implementation
|
||||||
|
|
||||||
|
`run.go` contains the public `Run` entrypoint and the main configuration orchestration path. Package-local run helpers are grouped by responsibility:
|
||||||
|
|
||||||
|
- `run_selection.go`: destination bundle selection, path mapping decisions, and fixed-path warnings;
|
||||||
|
- `run_warnings.go`: secret and SSH warning data;
|
||||||
|
- `run_output.go`: text plan lines, JSON action records, and output projections;
|
||||||
|
- `run_summary.go`: summary counters and JSON summary records;
|
||||||
|
- `run_failures.go`: destination failure aggregation and partial-result detection;
|
||||||
|
- `run_notify.go`: notification event projection and action filtering.
|
||||||
|
|
||||||
|
These helpers remain in `internal/app` because command output, warning collection, destination failure aggregation, notifier handoff, and backend construction are app-owned orchestration concerns.
|
||||||
|
|
||||||
## Backend and transform wiring
|
## Backend and transform wiring
|
||||||
|
|
||||||
The app-level backend factory registers local, SSH, and S3 backends for execution. S3 explicit credential references are resolved through the config environment resolver before adapter construction.
|
The app-level backend factory registers local, SSH, and S3 backends for execution. Source and destination backend config is converted through a shared app-local open spec before adapter construction. S3 explicit credential references are resolved through the config environment resolver.
|
||||||
|
|
||||||
The app-level transform registry registers Markdown-to-HTML using `internal/transform/markdown`. Lower-level publish code receives a resolver and does not import concrete transform implementations.
|
The app-level transform registry registers Markdown-to-HTML using `internal/transform/markdown`. Lower-level publish code receives a resolver and does not import concrete transform implementations.
|
||||||
|
|
||||||
@@ -47,7 +61,7 @@ Stdout write errors are returned immediately because the caller's requested outp
|
|||||||
|
|
||||||
`internal/app` coordinates packages but does not own manifest validation rules, destination state comparison, storage path rules, output planning, transform rendering, or backend-specific filesystem behavior.
|
`internal/app` coordinates packages but does not own manifest validation rules, destination state comparison, storage path rules, output planning, transform rendering, or backend-specific filesystem behavior.
|
||||||
|
|
||||||
`Validate` and `Inspect` are local path commands. Remote execution wiring currently belongs to `Run`.
|
Configured-source `Validate` and `Inspect` share source backend construction with `Run` and do not open destinations.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
`internal/bundle` parses, discovers, and validates source bundles through the storage interface.
|
`internal/bundle` discovers and validates source bundles through the storage interface. The source manifest model, manifest parsing, manifest validation, path rules, digest calculation, and producer-side local writer come from `pkg/bundle` so producer-facing APIs and distributor validation share one manifest contract.
|
||||||
|
|
||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
|
|
||||||
@@ -20,11 +20,13 @@ The source manifest requires:
|
|||||||
|
|
||||||
Each file requires `path`, `sha256`, and `size`. Digests must use lowercase `sha256:<64 hex>` format. `created` must parse as RFC3339.
|
Each file requires `path`, `sha256`, and `size`. Digests must use lowercase `sha256:<64 hex>` format. `created` must parse as RFC3339.
|
||||||
|
|
||||||
|
`pkg/bundle.ValidateDigest` is the canonical digest format validator for producer-facing and internal code. `internal/bundle.ValidateDigest` delegates to that public validator so source manifests and destination state use the same digest grammar.
|
||||||
|
|
||||||
## Validation
|
## Validation
|
||||||
|
|
||||||
`ValidateManifest` owns normalized source manifest semantics: schema version, id, digest format, timestamp presence, file list presence, source path safety, duplicate file paths, reserved paths, file digest format, non-negative file sizes, and the top-level bundle digest.
|
`pkg/bundle.ValidateManifest` owns normalized source manifest semantics: schema version, id, digest format, timestamp presence, file list presence, source path safety, duplicate file paths, reserved paths, file digest format, non-negative file sizes, and the top-level bundle digest.
|
||||||
|
|
||||||
Storage-backed bundle validation additionally checks file existence, regular-file type, file size, and per-file SHA-256.
|
Storage-backed bundle validation in `internal/bundle` additionally checks file existence, regular-file type, file size, and per-file SHA-256 for configured storage backends.
|
||||||
|
|
||||||
The bundle digest is SHA-256 of a deterministic JSON array of file records in manifest order with fields `path`, `sha256`, and `size`.
|
The bundle digest is SHA-256 of a deterministic JSON array of file records in manifest order with fields `path`, `sha256`, and `size`.
|
||||||
|
|
||||||
@@ -38,11 +40,11 @@ Manifest parsing and validation fail before destination planning. Storage-backed
|
|||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
Bundle code uses `internal/storage` and does not import concrete adapters. CLI local path support is wired in `internal/app`.
|
Internal bundle discovery uses `internal/storage` and does not import concrete adapters. Producer-side local filesystem manifest building, complete bundle writing, and validation belong to `pkg/bundle`. CLI local path support is wired in `internal/app`.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
Before changing bundle behavior, inspect tests under `internal/bundle`.
|
Before changing bundle behavior, inspect tests under `pkg/bundle` and `internal/bundle`.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,9 @@ Defaults are applied in `ApplyDefaults`:
|
|||||||
- SSH backend `port` defaults to `22`;
|
- SSH backend `port` defaults to `22`;
|
||||||
- SSH backend `host_key_policy` defaults to `accept-new`;
|
- SSH backend `host_key_policy` defaults to `accept-new`;
|
||||||
- destination publish policy defaults to source output only;
|
- destination publish policy defaults to source output only;
|
||||||
|
- Markdown-to-HTML mode defaults to `sidecar` when a transform block is present and mode is omitted;
|
||||||
|
- destination path mapping defaults to `preserve_relative`;
|
||||||
|
- destination link primary policy defaults to `auto` when a `links` block is present;
|
||||||
- `transfer.on_destination_same` defaults to `skip`;
|
- `transfer.on_destination_same` defaults to `skip`;
|
||||||
- `transfer.on_destination_older` defaults to `replace`;
|
- `transfer.on_destination_older` defaults to `replace`;
|
||||||
- `transfer.on_destination_newer` defaults to `skip`;
|
- `transfer.on_destination_newer` defaults to `skip`;
|
||||||
@@ -31,11 +34,15 @@ Defaults are applied in `ApplyDefaults`:
|
|||||||
|
|
||||||
## Validation responsibilities
|
## Validation responsibilities
|
||||||
|
|
||||||
Validation requires at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, and valid transfer actions.
|
Validation requires at least one pipeline, slug-like unique pipeline ids, one source per pipeline, at least one destination, slug-like unique destination ids within each pipeline, backend-specific required fields, valid validation policy, valid publish and transform combinations, valid destination path mapping mode, valid destination link config, and valid transfer actions.
|
||||||
|
|
||||||
Transfer validation accepts `replace` for `on_destination_newer` and `on_conflict`, but publish planning honors those destructive actions only when the current run explicitly requests force.
|
Transfer validation accepts `replace` for `on_destination_newer` and `on_conflict`, but publish planning honors those destructive actions only when the current run explicitly requests force.
|
||||||
|
|
||||||
`ValidatePublishTransformPolicy` is shared with publish planning so destination policy combinations are checked consistently. Publishing HTML requires an enabled Markdown-to-HTML transform in `sidecar` mode. A publish policy must select source output, HTML output, or both.
|
`ValidatePublishTransformPolicy` is shared with publish planning so destination policy combinations are checked consistently. Publishing HTML requires an enabled Markdown-to-HTML transform in `sidecar` or `index` mode. Enabled Markdown-to-HTML config is rejected when `publish.html` is false. `input` is accepted only for enabled `index` mode. A publish policy must select source output, HTML output, or both.
|
||||||
|
|
||||||
|
Destination path mapping accepts `preserve_relative` and `fixed`. The app layer applies the mapping when it selects destination bundle paths; config owns only YAML shape, defaulting, and validation.
|
||||||
|
|
||||||
|
Destination links are optional. When a `links` block is present, `base_url` is required, must use `http` or `https`, and must not include a query string or fragment. `primary` accepts `auto`, `html`, and `source`.
|
||||||
|
|
||||||
## Executable support boundary
|
## Executable support boundary
|
||||||
|
|
||||||
|
|||||||
27
docs/internal/link.md
Normal file
27
docs/internal/link.md
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Link URL Policy
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`internal/link` defines shared validation for configured and persisted HTTP link URLs.
|
||||||
|
|
||||||
|
## Inputs and outputs
|
||||||
|
|
||||||
|
Input is a URL string. Output is either nil for an accepted URL or a concise validation error that callers wrap with field context.
|
||||||
|
|
||||||
|
## Validation behavior
|
||||||
|
|
||||||
|
Accepted URLs must parse successfully, use `http` or `https`, include a host, and omit query strings and fragments.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
This package validates URL shape only. It does not construct destination output URLs, choose primary URLs, infer public URLs from backend configuration, or read configuration files.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Before changing link URL policy, inspect tests under `internal/link` and callers in `internal/config`, `internal/state`, and `internal/publish`.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
- Configured `links.base_url`, persisted `links.primary_url`, persisted output `url`, and publish link planning use the same URL policy.
|
||||||
|
- Callers own field-specific error context.
|
||||||
|
- URL path construction remains in `internal/publish`.
|
||||||
@@ -6,9 +6,9 @@
|
|||||||
|
|
||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
|
|
||||||
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, publish policy, transform policy, transformer resolver, transfer policy, destination bundle path, existing destination state, and whether explicit force was requested for the current run.
|
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, publish policy, transform policy, optional link policy, transformer resolver, transfer policy, path mapping mode, destination bundle path, existing destination state, and whether explicit force was requested for the current run.
|
||||||
|
|
||||||
Output is a plan with an action, reason, and selected source or generated outputs. Execution writes selected source files, generated files, and `.distributor.json` for publish or replacement actions.
|
Output is a plan with an action, reason, optional primary URL, and selected source or generated outputs. Execution writes selected source files, generated files, and `.distributor.json` for publish or replacement actions.
|
||||||
|
|
||||||
## Actions
|
## Actions
|
||||||
|
|
||||||
@@ -22,9 +22,15 @@ Execution fails if a write, delete, state serialization, or context check fails.
|
|||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
The current implementation publishes source files and Markdown-to-HTML sidecar outputs. Backend behavior is supplied through `internal/storage`; app runtime currently supplies local, SSH, and S3 backends.
|
The package publishes source files and Markdown-to-HTML outputs. Markdown sidecar mode writes same-directory `.html` outputs, and Markdown index mode writes `index.html`. Backend behavior is supplied through `internal/storage`; app runtime supplies local, SSH, and S3 backends.
|
||||||
|
|
||||||
The package uses `internal/state` for destination comparison, `internal/storage` for IO, and the shared `internal/config` publish/transform policy helper for request validation. It resolves transforms through a narrow resolver supplied by the caller; concrete transform registration is owned by the app layer. It does not parse CLI flags or load config files.
|
The package uses `internal/state` for destination comparison, `internal/storage` for IO, and the shared `internal/config` publish/transform policy helper for request validation. It resolves transforms through a narrow resolver supplied by the caller; concrete transform registration is owned by the app layer. It does not parse CLI flags, load config files, or choose which source bundles a destination receives.
|
||||||
|
|
||||||
|
The package owns projection from planned publish outputs to destination state output records and managed destination output paths. App JSON results and notification events keep their own schemas, but may use the publish output projection to avoid field-mapping drift.
|
||||||
|
|
||||||
|
The app layer computes the destination bundle path before planning. `preserve_relative` destinations pass the source-root-relative bundle path. `fixed` destinations pass an empty destination bundle path, which means the destination backend root, and pass only the newest selected source bundle for that destination.
|
||||||
|
|
||||||
|
When link config is present, publish planning builds per-output URLs from `links.base_url`, the destination bundle path, and each output path. `index.html` outputs use directory-style URLs. The primary URL is selected from planned outputs according to the destination primary policy.
|
||||||
|
|
||||||
## Safety
|
## Safety
|
||||||
|
|
||||||
@@ -37,6 +43,8 @@ Before changing publish behavior, inspect tests under `internal/publish` and run
|
|||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- Publish planning is deterministic for the same source, destination state, policies, and transform outputs.
|
- Publish planning is deterministic for the same source, destination state, policies, and transform outputs.
|
||||||
|
- Destination bundle paths are caller-supplied and are interpreted relative to the destination backend root.
|
||||||
|
- URL generation uses URL path semantics and does not infer public URLs from backend configuration.
|
||||||
- Normal replacement deletes only managed paths recorded in existing state plus `.distributor.json`.
|
- Normal replacement deletes only managed paths recorded in existing state plus `.distributor.json`.
|
||||||
- Forced replacement is explicit per run and deletes only within the destination bundle path.
|
- Forced replacement is explicit per run and deletes only within the destination bundle path.
|
||||||
- Publish execution writes destination state after selected outputs are written.
|
- Publish execution writes destination state after selected outputs are written.
|
||||||
|
|||||||
@@ -19,13 +19,15 @@ Input is JSON destination state plus the current source manifest, pipeline id, d
|
|||||||
- `source.manifest`
|
- `source.manifest`
|
||||||
- `outputs`
|
- `outputs`
|
||||||
|
|
||||||
`distributor_version` is optional diagnostic metadata. `published_at` parses as RFC3339 and distributor-written state serializes it as RFC3339 UTC.
|
`distributor_version` is optional diagnostic metadata. `links` is optional URL metadata. `published_at` parses as RFC3339 and distributor-written state serializes it as RFC3339 UTC.
|
||||||
|
|
||||||
The embedded `source.manifest` is validated with the same source manifest rules used by `internal/bundle`.
|
The embedded `source.manifest` is validated with the same source manifest rules used by `internal/bundle`.
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
Each output records `path`, `kind`, `source_path`, `sha256`, and `size`. Supported output kinds are `source` and `generated`. Generated outputs require `transform`.
|
Each output records `path`, `kind`, `source_path`, `sha256`, and `size`. Supported output kinds are `source` and `generated`. Generated outputs require `transform`. Outputs may record `url` when the destination has link generation configured.
|
||||||
|
|
||||||
|
The optional top-level `links.primary_url` records the selected primary URL for the published destination bundle. It is omitted when link generation is not configured or when the destination primary policy has no matching output.
|
||||||
|
|
||||||
## Comparison
|
## Comparison
|
||||||
|
|
||||||
@@ -33,7 +35,7 @@ Comparison outcomes cover absent destination state, unmanaged destination conten
|
|||||||
|
|
||||||
## Failure behavior
|
## Failure behavior
|
||||||
|
|
||||||
Invalid JSON, invalid state schema, invalid embedded source manifests, unsafe output paths, unsupported output kinds, missing generated-output transform names, and mismatched pipeline or destination ids produce comparison outcomes that publish planning can turn into fail actions. Supported identity and source-manifest conflicts can become forced replacement only when publish planning receives explicit force and compatible transfer policy.
|
Invalid JSON, invalid state schema, invalid embedded source manifests, unsafe output paths, invalid stored URLs, unsupported output kinds, missing generated-output transform names, and mismatched pipeline or destination ids produce comparison outcomes that publish planning can turn into fail actions. Supported identity and source-manifest conflicts can become forced replacement only when publish planning receives explicit force and compatible transfer policy.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
@@ -48,5 +50,6 @@ Before changing destination state behavior, inspect tests under `internal/state`
|
|||||||
- `.distributor.json` is the destination sentinel and state record.
|
- `.distributor.json` is the destination sentinel and state record.
|
||||||
- Embedded source manifests use the same validation rules as source bundles.
|
- Embedded source manifests use the same validation rules as source bundles.
|
||||||
- Generated outputs always record a transform id.
|
- Generated outputs always record a transform id.
|
||||||
|
- Stored URLs are optional and must be absolute HTTP or HTTPS URLs when present.
|
||||||
- Comparison returns outcomes and reasons; it does not mutate storage.
|
- Comparison returns outcomes and reasons; it does not mutate storage.
|
||||||
- `distributor_version` is diagnostic metadata, not a comparison key.
|
- `distributor_version` is diagnostic metadata, not a comparison key.
|
||||||
|
|||||||
@@ -26,6 +26,19 @@ Storage errors use typed categories such as not found, already exists, invalid p
|
|||||||
|
|
||||||
Backends may wrap implementation-specific errors, but callers should receive storage errors where practical. Traversal can stop cleanly with `ErrStopWalk`.
|
Backends may wrap implementation-specific errors, but callers should receive storage errors where practical. Traversal can stop cleanly with `ErrStopWalk`.
|
||||||
|
|
||||||
|
## Traversal helpers
|
||||||
|
|
||||||
|
Backends own their traversal mechanics. The local adapter owns filesystem walking, the SSH adapter owns SFTP directory walking, and the S3 adapter owns object listing and pagination.
|
||||||
|
|
||||||
|
`internal/storage` owns the shared callback emission rules used by backends:
|
||||||
|
|
||||||
|
- context cancellation is checked before callback emission;
|
||||||
|
- `WalkOptions.Limit` bounds the number of emitted entries;
|
||||||
|
- `ErrStopWalk` stops traversal without becoming a caller-visible error;
|
||||||
|
- callback errors are wrapped as storage walk errors.
|
||||||
|
|
||||||
|
`storage.HasAny(ctx, backend, prefix)` provides the shared destination-content check. It calls `Walk` with non-recursive, limit-one traversal and stops after the first emitted entry.
|
||||||
|
|
||||||
## Deletion
|
## Deletion
|
||||||
|
|
||||||
`DeleteManagedBundle` may delete listed managed outputs plus `.distributor.json`.
|
`DeleteManagedBundle` may delete listed managed outputs plus `.distributor.json`.
|
||||||
|
|||||||
@@ -2,11 +2,11 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
`internal/transform` defines generated publication artifacts. `internal/transform/markdown` implements Markdown-to-HTML sidecar generation.
|
`internal/transform` defines generated publication artifacts. `internal/transform/markdown` implements Markdown-to-HTML generation.
|
||||||
|
|
||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
|
|
||||||
Inputs are a validated source bundle and source backend. Outputs include destination path, source path, transform id, generated bytes, SHA-256, and size.
|
Inputs are a validated source bundle, source backend, and transform options supplied by publish planning. Outputs include destination path, source path, transform id, generated bytes, SHA-256, and size.
|
||||||
|
|
||||||
## Registry
|
## Registry
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ Inputs are a validated source bundle and source backend. Outputs include destina
|
|||||||
|
|
||||||
## Markdown behavior
|
## Markdown behavior
|
||||||
|
|
||||||
Markdown files ending in `.md` generate `.html` files in the same logical directory. Non-Markdown files do not generate outputs. Raw HTML embedded in Markdown is not passed through by the renderer.
|
Markdown sidecar mode renders files ending in `.md` to `.html` files in the same logical directory. Markdown index mode renders one selected manifest-listed Markdown file to `index.html`. Non-Markdown files do not generate sidecar outputs. Raw HTML embedded in Markdown is not passed through by the renderer.
|
||||||
|
|
||||||
Generated HTML is deterministic for the same source content and transform configuration.
|
Generated HTML is deterministic for the same source content and transform configuration.
|
||||||
|
|
||||||
@@ -22,7 +22,7 @@ See `docs/integrations/markdown.md` for the Goldmark integration contract.
|
|||||||
|
|
||||||
## Failure behavior
|
## Failure behavior
|
||||||
|
|
||||||
Transform resolution fails when a requested transform id is not registered. Markdown rendering fails when the source file cannot be read or rendered. Publish planning fails when HTML output is requested and the selected transform produces no outputs for a bundle.
|
Transform resolution fails when a requested transform id is not registered. Markdown rendering fails when the source file cannot be read or rendered. Index input selection fails when the configured input is unsafe, not listed, not Markdown, or when no configured input can be inferred from exactly one manifest-listed Markdown file. Publish planning fails when HTML output is requested and the selected transform produces no outputs for a bundle.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
@@ -42,5 +42,6 @@ Before changing transform behavior, inspect tests under:
|
|||||||
- Source bundle files are never mutated by transforms.
|
- Source bundle files are never mutated by transforms.
|
||||||
- Generated outputs record destination path, source path, transform id, SHA-256, and size.
|
- Generated outputs record destination path, source path, transform id, SHA-256, and size.
|
||||||
- Markdown sidecar naming changes only the `.md` extension to `.html`.
|
- Markdown sidecar naming changes only the `.md` extension to `.html`.
|
||||||
|
- Markdown index mode always writes `index.html`.
|
||||||
- Non-Markdown source files do not generate Markdown outputs.
|
- Non-Markdown source files do not generate Markdown outputs.
|
||||||
- Transform registration stays outside publish planning.
|
- Transform registration stays outside publish planning.
|
||||||
|
|||||||
@@ -26,12 +26,30 @@ Run the local HTML publication:
|
|||||||
go run ./cmd/distributor run --config examples/local-html.yml
|
go run ./cmd/distributor run --config examples/local-html.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Run the local `index.html` publication:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-index.yml
|
||||||
|
```
|
||||||
|
|
||||||
Preview local fan-out publication:
|
Preview local fan-out publication:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go run ./cmd/distributor run --config examples/fan-out.yml --dry-run
|
go run ./cmd/distributor run --config examples/fan-out.yml --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Preview local archive-plus-latest publication:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/archive-and-latest.yml --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Preview a run for automation:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/fan-out.yml --dry-run --format json
|
||||||
|
```
|
||||||
|
|
||||||
Preview an environment-gated SSH destination config after editing it for an SSH/SFTP endpoint you control:
|
Preview an environment-gated SSH destination config after editing it for an SSH/SFTP endpoint you control:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -44,11 +62,21 @@ Preview an environment-gated S3 destination config after editing it for an S3-co
|
|||||||
go run ./cmd/distributor run --config examples/s3-destination.yml --dry-run
|
go run ./cmd/distributor run --config examples/s3-destination.yml --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Validate one configured source without opening destinations:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor validate --config examples/local-publish.yml --pipeline example-source-bundle
|
||||||
|
```
|
||||||
|
|
||||||
## Filesystem Layout
|
## Filesystem Layout
|
||||||
|
|
||||||
Source bundles are discovered beneath the configured source root. Each bundle is a directory containing `manifest.json`.
|
Source bundles are discovered beneath the configured source root. Each bundle is a directory containing `manifest.json`.
|
||||||
|
|
||||||
Destination bundle paths preserve the source bundle path relative to the source root. A source bundle at the source root publishes to the destination root. A source bundle under `daily/` publishes under `daily/` at each destination.
|
Destination bundle paths are configured per destination with `path_mapping.mode`.
|
||||||
|
|
||||||
|
The default mode, `preserve_relative`, preserves the source bundle path relative to the source root. A source bundle at the source root publishes to the destination root. A source bundle under `daily/` publishes under `daily/` at that destination.
|
||||||
|
|
||||||
|
The `fixed` mode publishes one selected source bundle at the destination backend root. A fixed destination with local `path: /srv/www/reports/latest` writes outputs and `.distributor.json` directly under `/srv/www/reports/latest`. Fixed destinations select the newest discovered source bundle by manifest `created` timestamp, with the source-root-relative bundle path as the deterministic tie-breaker.
|
||||||
|
|
||||||
The maintained local examples write under `workspace/`, which is ignored by Git.
|
The maintained local examples write under `workspace/`, which is ignored by Git.
|
||||||
|
|
||||||
@@ -64,19 +92,173 @@ Each published destination bundle contains `.distributor.json`. This file is the
|
|||||||
- publication timestamp;
|
- publication timestamp;
|
||||||
- source manifest used for publication;
|
- source manifest used for publication;
|
||||||
- copied source output metadata;
|
- copied source output metadata;
|
||||||
- generated output metadata.
|
- generated output metadata;
|
||||||
|
- optional public URL metadata when destination links are configured.
|
||||||
|
|
||||||
`manifest.json` from the source bundle is not copied as destination state.
|
`manifest.json` from the source bundle is not copied as destination state.
|
||||||
|
|
||||||
Do not edit `.distributor.json` by hand during normal operation. If it is missing or invalid while destination files remain, `distributor` treats the destination as unmanaged or conflicted.
|
Do not edit `.distributor.json` by hand during normal operation. If it is missing or invalid while destination files remain, `distributor` treats the destination as unmanaged or conflicted.
|
||||||
|
|
||||||
|
## Go Producer Bundles
|
||||||
|
|
||||||
|
Go producer applications can import `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to create complete local source bundles with the same path, digest, timestamp, and validation rules used by `distributor`. The package also exposes digest helpers, including `ValidateDigest`, for producer code that needs to validate lowercase `sha256:<64 hex>` strings before writing manifests.
|
||||||
|
|
||||||
|
Minimal producer-side bundle creation:
|
||||||
|
|
||||||
|
```go
|
||||||
|
manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{
|
||||||
|
Root: outputDir,
|
||||||
|
ID: "reports.example.2026-05-30",
|
||||||
|
Files: []bundle.BundleFile{
|
||||||
|
{SourcePath: reportPath, Path: "report.md"},
|
||||||
|
{SourcePath: summaryPath, Path: "summary.txt"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`WriteBundle` copies local producer files into a sibling temporary directory, writes `manifest.json`, validates the result, and promotes the completed bundle into place. It fails if `Root` already exists unless `Overwrite` is true. With overwrite enabled, it builds and validates the replacement before moving the existing root aside.
|
||||||
|
|
||||||
|
Use `BuildManifest` and `WriteManifest` when a producer already wrote all bundle files into the final root. `BuildManifest` can preserve an explicit file order, or `Scan: true` can recursively include regular files under `Root` in deterministic slash-path order. Scan mode includes dotfiles, excludes files named `manifest.json` or `.distributor.json`, and rejects symlinks.
|
||||||
|
|
||||||
|
Shell producers can create the same manifest through the CLI after writing bundle files:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor manifest create <bundle-path> --id reports.example.2026-05-30
|
||||||
|
go run ./cmd/distributor validate <bundle-path>
|
||||||
|
```
|
||||||
|
|
||||||
|
Use repeated `--file` flags to preserve a specific file order. If no `--file` flags are provided, the command scans the bundle directory recursively using the same filtering rules as `pkg/bundle.BuildManifest`.
|
||||||
|
|
||||||
|
## Static HTML Publication
|
||||||
|
|
||||||
|
Markdown-to-HTML publication can write sidecar files or a fixed `index.html`.
|
||||||
|
|
||||||
|
Use sidecar mode when each Markdown source should keep a matching HTML filename:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: sidecar
|
||||||
|
```
|
||||||
|
|
||||||
|
Use index mode for static-site destinations that should serve a bundle through `index.html`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: index
|
||||||
|
input: report.md
|
||||||
|
```
|
||||||
|
|
||||||
|
If `input` is omitted in index mode, the source manifest must list exactly one Markdown file. Generated HTML is recorded in `.distributor.json` with `kind: generated`, `source_path`, `transform: markdown_to_html`, digest, and size metadata.
|
||||||
|
|
||||||
|
## Archive And Latest Fan-Out
|
||||||
|
|
||||||
|
A pipeline can publish the same source to an archive destination and a stable latest destination:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/distributor/reports
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /srv/reports/archive
|
||||||
|
path_mapping:
|
||||||
|
mode: preserve_relative
|
||||||
|
publish:
|
||||||
|
source: true
|
||||||
|
html: false
|
||||||
|
- id: latest-html
|
||||||
|
backend: local
|
||||||
|
path: /srv/www/reports/latest
|
||||||
|
path_mapping:
|
||||||
|
mode: fixed
|
||||||
|
links:
|
||||||
|
base_url: https://reports.example.com/latest
|
||||||
|
primary: auto
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: index
|
||||||
|
input: report.md
|
||||||
|
```
|
||||||
|
|
||||||
|
The archive destination plans every discovered source bundle at its source-relative path. The fixed latest destination plans only the newest discovered bundle and writes `index.html` plus `.distributor.json` at its backend root.
|
||||||
|
|
||||||
|
## Static Site URLs
|
||||||
|
|
||||||
|
Use destination `links` when a destination backend root corresponds to a public HTTP or HTTPS URL:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
links:
|
||||||
|
base_url: https://reports.example.com/archive
|
||||||
|
primary: auto
|
||||||
|
```
|
||||||
|
|
||||||
|
Distributor records URLs in `.distributor.json`; it does not publish notifications or infer URLs from local, SSH, or S3 backend fields.
|
||||||
|
|
||||||
|
For archive-style destinations, URLs include the destination bundle path. A source bundle under `daily/brentwood/2026-06-01` with `base_url: https://reports.example.com/archive` can produce:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://reports.example.com/archive/daily/brentwood/2026-06-01/report.html
|
||||||
|
```
|
||||||
|
|
||||||
|
For fixed destinations, URLs are rooted at `links.base_url`. A fixed HTML index destination with `base_url: https://reports.example.com/latest` records:
|
||||||
|
|
||||||
|
```text
|
||||||
|
https://reports.example.com/latest/
|
||||||
|
```
|
||||||
|
|
||||||
|
`index.html` outputs use directory-style URLs. Other outputs include their filename. The primary URL is selected from the published outputs using the destination `links.primary` policy.
|
||||||
|
|
||||||
|
## Source Validation and Inspection
|
||||||
|
|
||||||
|
`validate` and `inspect` can operate on a local path or on one configured pipeline source. Configured source mode requires both `--config` and `--pipeline`; it loads the normal config, resolves `secrets.directory`, opens only the selected source backend, and does not open any destinations.
|
||||||
|
|
||||||
|
Configured source validation is useful when producers write directly to SSH or S3 storage:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor validate --config <config-path> --pipeline <pipeline-id>
|
||||||
|
go run ./cmd/distributor inspect --config <config-path> --pipeline <pipeline-id>
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--bundle <path>` to validate or inspect one source-root-relative bundle directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor validate \
|
||||||
|
--config <config-path> \
|
||||||
|
--pipeline <pipeline-id> \
|
||||||
|
--bundle daily/2026-06-01
|
||||||
|
```
|
||||||
|
|
||||||
|
For configured SSH sources, host key and authentication behavior matches `run`. For configured S3 sources, endpoint, bucket, prefix, region, path-style, explicit credential environment variables, and `secrets.directory` handling match `run`.
|
||||||
|
|
||||||
## Dry Runs
|
## Dry Runs
|
||||||
|
|
||||||
`--dry-run` loads and validates config, discovers source bundles, inspects destination state, plans outputs, and prints summary lines. It does not write output files, destination state, or SSH `known_hosts` entries.
|
`--dry-run` loads and validates config, discovers source bundles, inspects destination state, plans outputs, and prints summary lines. It does not write output files, destination state, or SSH `known_hosts` entries.
|
||||||
|
|
||||||
Dry-run output is useful before publishing to confirm actions such as `publish_new`, `replace_older`, `force_replace`, `skip_same`, and `skip_destination_newer`.
|
Dry-run output is useful before publishing to confirm actions such as `publish_new`, `replace_older`, `force_replace`, `skip_same`, and `skip_destination_newer`.
|
||||||
|
|
||||||
Destination action lines include the destination backend, so mixed local, SSH, and S3 fan-out runs can be audited before publication.
|
Destination action lines include the destination backend, so mixed local, SSH, and S3 fan-out runs can be audited before publication. Fixed path destinations add `path_mapping=fixed target=.` to planned action lines. Dry-run also prints a warning with the fixed destination candidate count and selected source bundle; destructive fixed replacements print an additional warning that the destination root would be replaced.
|
||||||
|
|
||||||
|
Use `--format json` when another process needs stable run data. JSON output includes warnings, pipeline summaries, destination actions, destination bundle paths, path mapping modes, optional link URLs, output records, final counters, and partial failure records. The summary includes `fixed_path`. If one destination fails after planning or execution begins, JSON output still contains the successful and failed destination records with `ok: false`, and the command exits non-zero.
|
||||||
|
|
||||||
## Retry and Replacement Behavior
|
## Retry and Replacement Behavior
|
||||||
|
|
||||||
@@ -99,7 +281,7 @@ go run ./cmd/distributor run --config <config-path> --force
|
|||||||
|
|
||||||
Forced replacement can overwrite unmanaged non-empty destination paths. Destination state conflicts require `transfer.on_conflict: replace` plus `--force`. Newer destination state requires `transfer.on_destination_newer: replace` plus `--force`.
|
Forced replacement can overwrite unmanaged non-empty destination paths. Destination state conflicts require `transfer.on_conflict: replace` plus `--force`. Newer destination state requires `transfer.on_destination_newer: replace` plus `--force`.
|
||||||
|
|
||||||
Forced replacement deletes the current destination bundle path before writing outputs and state. It does not delete above that bundle path. For S3 destinations, deletion is constrained to the configured bucket and prefix plus the destination bundle prefix. Force is per run only and has no config default.
|
Forced replacement deletes the current destination bundle path before writing outputs and state. It does not delete above that bundle path. For fixed destinations, the destination bundle path is the backend root, so forced replacement may clear that configured root but not its parent path, sibling directories, or anything outside the configured S3 bucket and prefix. Force is per run only and has no config default.
|
||||||
|
|
||||||
## Failure Handling
|
## Failure Handling
|
||||||
|
|
||||||
@@ -107,6 +289,8 @@ If one destination fails in a fan-out run, independent later destinations are st
|
|||||||
|
|
||||||
Errors include the pipeline id, destination id, destination backend, and bundle path where applicable.
|
Errors include the pipeline id, destination id, destination backend, and bundle path where applicable.
|
||||||
|
|
||||||
|
In JSON mode, destination failures after planning or execution begins are reported in the top-level `errors` array and in the run result while preserving a non-zero exit code. Fatal setup errors such as an unreadable config or invalid secrets directory write no JSON document.
|
||||||
|
|
||||||
If a write fails during publication, `distributor` attempts to remove outputs written during that failed attempt so a retry does not see those partial outputs as unmanaged destination content.
|
If a write fails during publication, `distributor` attempts to remove outputs written during that failed attempt so a retry does not see those partial outputs as unmanaged destination content.
|
||||||
|
|
||||||
After a successful publish or replacement, the internal notifier hook runs. The current default notifier is a no-op. Skipped destinations do not invoke it.
|
After a successful publish or replacement, the internal notifier hook runs. The current default notifier is a no-op. Skipped destinations do not invoke it.
|
||||||
@@ -127,7 +311,7 @@ S3 execution uses the AWS SDK for Go v2. Configure `endpoint`, `bucket`, optiona
|
|||||||
|
|
||||||
When explicit credential env names are configured, both variables must resolve to non-empty values through the real process environment or `secrets.directory`. When they are omitted, the AWS SDK default credential chain is used as-is.
|
When explicit credential env names are configured, both variables must resolve to non-empty values through the real process environment or `secrets.directory`. When they are omitted, the AWS SDK default credential chain is used as-is.
|
||||||
|
|
||||||
Normal replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Forced replacement deletes objects under the bounded destination bundle prefix. Distributor does not manage bucket versioning or delete markers.
|
Normal replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Forced replacement deletes objects under the bounded destination bundle prefix. For fixed destinations, that prefix is the configured bucket plus optional `prefix`. Distributor does not manage bucket versioning or delete markers.
|
||||||
|
|
||||||
## Secrets Directory
|
## Secrets Directory
|
||||||
|
|
||||||
@@ -138,9 +322,9 @@ secrets:
|
|||||||
directory: /run/secrets/distributor
|
directory: /run/secrets/distributor
|
||||||
```
|
```
|
||||||
|
|
||||||
The directory is loaded during `run` before any source or destination backend is opened. If the directory is missing, unreadable, or contains an invalid secret filename, the run fails before publication work starts.
|
The directory is loaded during `run` and configured-source `validate` or `inspect` before any backend is opened. If the directory is missing, unreadable, or contains an invalid secret filename, the command fails before storage work starts.
|
||||||
|
|
||||||
Real process environment values take precedence over files with the same name. If the values differ and stdout is enabled, `run` prints a warning naming the ignored secret file variable without printing either value. The process environment is not changed.
|
Real process environment values take precedence over files with the same name. If the values differ and stdout is enabled, `run` and configured-source diagnostics print a warning naming the ignored secret file variable without printing either value. The process environment is not changed.
|
||||||
|
|
||||||
## Caveats
|
## Caveats
|
||||||
|
|
||||||
|
|||||||
@@ -22,19 +22,21 @@ The current core workflow is:
|
|||||||
2. open the source backend;
|
2. open the source backend;
|
||||||
3. discover source bundles beneath the source root;
|
3. discover source bundles beneath the source root;
|
||||||
4. validate each source bundle and its `manifest.json`;
|
4. validate each source bundle and its `manifest.json`;
|
||||||
5. for each configured destination, inspect destination state;
|
5. select the source bundle or bundles for each destination according to that destination's path mapping policy;
|
||||||
6. compare source state to destination state;
|
6. open each destination backend independently;
|
||||||
7. build a publish plan;
|
7. inspect destination state at the resolved destination bundle path;
|
||||||
8. optionally transform Markdown to HTML for that destination;
|
8. compare source state to destination state;
|
||||||
9. publish selected source and generated artifacts;
|
9. build a publish plan that selects source files, generated files, destination paths, and optional public URLs;
|
||||||
10. write `.distributor.json` as the destination sentinel/state file;
|
10. optionally transform Markdown to HTML for that destination;
|
||||||
11. run the notification hook, which is a no-op in the MVP.
|
11. publish selected source and generated artifacts;
|
||||||
|
12. write `.distributor.json` as the destination sentinel/state file;
|
||||||
|
13. run the notification hook, whose default implementation is currently a no-op.
|
||||||
|
|
||||||
## Pipeline Model
|
## Pipeline Model
|
||||||
|
|
||||||
A pipeline has exactly one source and one or more destinations.
|
A pipeline has exactly one source and one or more destinations.
|
||||||
|
|
||||||
The source is discovered and validated once. Each destination has independent backend configuration, publication policy, transform policy, replacement behavior, state, and notification behavior.
|
The source is discovered and validated once. Each destination has independent backend configuration, path mapping, publication policy, transform policy, public link policy, replacement behavior, state, and notification behavior.
|
||||||
|
|
||||||
The pipeline model is fan-out by design:
|
The pipeline model is fan-out by design:
|
||||||
|
|
||||||
@@ -53,7 +55,7 @@ A source bundle is a directory containing `manifest.json`.
|
|||||||
|
|
||||||
`manifest.json` is the sole producer-to-`distributor` contract. `distributor` must not rely on producer-specific work directory layouts, filenames, metadata, or conventions outside the configured source root and the source manifest.
|
`manifest.json` is the sole producer-to-`distributor` contract. `distributor` must not rely on producer-specific work directory layouts, filenames, metadata, or conventions outside the configured source root and the source manifest.
|
||||||
|
|
||||||
The MVP source manifest schema is intentionally minimal:
|
The source manifest schema is intentionally minimal:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -73,7 +75,7 @@ The MVP source manifest schema is intentionally minimal:
|
|||||||
|
|
||||||
Required fields:
|
Required fields:
|
||||||
|
|
||||||
- `schema_version`: source manifest schema version. MVP value: `1`.
|
- `schema_version`: source manifest schema version. Current value: `1`.
|
||||||
- `id`: stable bundle identifier.
|
- `id`: stable bundle identifier.
|
||||||
- `digest`: SHA-256 digest for the listed files.
|
- `digest`: SHA-256 digest for the listed files.
|
||||||
- `created`: RFC3339 timestamp. UTC is preferred; explicit offsets are allowed.
|
- `created`: RFC3339 timestamp. UTC is preferred; explicit offsets are allowed.
|
||||||
@@ -101,6 +103,7 @@ Each destination bundle path is managed by `.distributor.json`. This file is bot
|
|||||||
- the normalized source manifest used for publication;
|
- the normalized source manifest used for publication;
|
||||||
- metadata for copied source outputs;
|
- metadata for copied source outputs;
|
||||||
- metadata for generated outputs, such as HTML files;
|
- metadata for generated outputs, such as HTML files;
|
||||||
|
- optional URL metadata for published outputs;
|
||||||
- any additional metadata required by `distributor`.
|
- any additional metadata required by `distributor`.
|
||||||
|
|
||||||
A representative destination state file is:
|
A representative destination state file is:
|
||||||
@@ -127,14 +130,18 @@ A representative destination state file is:
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"links": {
|
||||||
|
"primary_url": "https://reports.example.com/weather-daily/"
|
||||||
|
},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"path": "report.html",
|
"path": "index.html",
|
||||||
"kind": "generated",
|
"kind": "generated",
|
||||||
"source_path": "report.md",
|
"source_path": "report.md",
|
||||||
"transform": "markdown_to_html",
|
"transform": "markdown_to_html",
|
||||||
"sha256": "sha256:...",
|
"sha256": "sha256:...",
|
||||||
"size": 23456
|
"size": 23456,
|
||||||
|
"url": "https://reports.example.com/weather-daily/"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -157,6 +164,8 @@ Transforms are configured per destination. A destination may receive source file
|
|||||||
|
|
||||||
The MVP supports only Markdown-to-HTML transformation. HTML generation must not mutate the source bundle. Generated outputs must be deterministic from the source bundle and destination transform configuration, and must be recorded in `.distributor.json`.
|
The MVP supports only Markdown-to-HTML transformation. HTML generation must not mutate the source bundle. Generated outputs must be deterministic from the source bundle and destination transform configuration, and must be recorded in `.distributor.json`.
|
||||||
|
|
||||||
|
Destination path mapping and public link generation are destination behavior. Source manifests do not declare where a bundle is published or which public URLs are recorded.
|
||||||
|
|
||||||
The application should distinguish:
|
The application should distinguish:
|
||||||
|
|
||||||
- transform policy: how derived files are generated;
|
- transform policy: how derived files are generated;
|
||||||
@@ -187,11 +196,13 @@ Avoid dependencies for small conveniences. Do not let external dependency types
|
|||||||
Use this current layout unless the project has a documented reason to differ:
|
Use this current layout unless the project has a documented reason to differ:
|
||||||
|
|
||||||
- `cmd/distributor`: application entrypoint only.
|
- `cmd/distributor`: application entrypoint only.
|
||||||
|
- `pkg/bundle`: public producer-facing source manifest model, digest logic, parsing, manifest building, complete local bundle writing, and local validation helpers.
|
||||||
- `internal/app`: application orchestration and top-level use cases.
|
- `internal/app`: application orchestration and top-level use cases.
|
||||||
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
|
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
|
||||||
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
|
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
|
||||||
- `internal/bundle`: source manifest parsing, source bundle discovery, source digest validation, and source bundle model.
|
- `internal/bundle`: storage-backed source bundle discovery and validation over the public manifest contract.
|
||||||
- `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
|
- `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
|
||||||
|
- `internal/link`: shared HTTP URL validation for configured and persisted link metadata.
|
||||||
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
|
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
|
||||||
- `internal/adapters/local`: local filesystem backend.
|
- `internal/adapters/local`: local filesystem backend.
|
||||||
- `internal/adapters/ssh`: SSH/SFTP backend.
|
- `internal/adapters/ssh`: SSH/SFTP backend.
|
||||||
@@ -228,17 +239,20 @@ Pipeline configuration should express:
|
|||||||
- pipeline id;
|
- pipeline id;
|
||||||
- one source backend;
|
- one source backend;
|
||||||
- one or more destinations;
|
- one or more destinations;
|
||||||
|
- per-destination path mapping;
|
||||||
- per-destination publish policy;
|
- per-destination publish policy;
|
||||||
- per-destination transform policy;
|
- per-destination transform policy;
|
||||||
|
- per-destination public link policy;
|
||||||
- validation behavior;
|
- validation behavior;
|
||||||
- destination conflict/replacement behavior.
|
- destination conflict/replacement behavior.
|
||||||
|
|
||||||
## Modules, Stages, and Registries
|
## Modules and Registries
|
||||||
|
|
||||||
Each major stage should have an explicit input/output contract:
|
Each major workflow step should have an explicit input/output contract:
|
||||||
|
|
||||||
- source discovery;
|
- source discovery;
|
||||||
- source validation;
|
- source validation;
|
||||||
|
- destination bundle selection;
|
||||||
- destination state inspection;
|
- destination state inspection;
|
||||||
- destination comparison;
|
- destination comparison;
|
||||||
- transform planning/execution;
|
- transform planning/execution;
|
||||||
@@ -297,8 +311,10 @@ Important tests include:
|
|||||||
- relative path safety and path traversal rejection;
|
- relative path safety and path traversal rejection;
|
||||||
- destination `.distributor.json` parsing and comparison;
|
- destination `.distributor.json` parsing and comparison;
|
||||||
- same/older/newer/conflict publish decisions;
|
- same/older/newer/conflict publish decisions;
|
||||||
|
- destination bundle path mapping;
|
||||||
- destructive replacement safety checks;
|
- destructive replacement safety checks;
|
||||||
- transform output planning and metadata recording;
|
- transform output planning and metadata recording;
|
||||||
|
- public URL planning and state metadata;
|
||||||
- dry-run output;
|
- dry-run output;
|
||||||
- local backend behavior with temporary directories;
|
- local backend behavior with temporary directories;
|
||||||
- fake backend behavior for storage-facing core logic.
|
- fake backend behavior for storage-facing core logic.
|
||||||
|
|||||||
@@ -6,10 +6,11 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
|
|||||||
## Repository Layout
|
## Repository Layout
|
||||||
|
|
||||||
- `cmd/distributor`: executable entrypoint only.
|
- `cmd/distributor`: executable entrypoint only.
|
||||||
|
- `pkg/bundle`: public producer-facing source manifest and local bundle writer helpers.
|
||||||
- `internal/app`: top-level use cases for `run`, `validate`, and `inspect`.
|
- `internal/app`: top-level use cases for `run`, `validate`, and `inspect`.
|
||||||
- `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
|
- `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
|
||||||
- `internal/config`: YAML configuration structs, loading, defaults, and validation.
|
- `internal/config`: YAML configuration structs, loading, defaults, and validation.
|
||||||
- `internal/bundle`: source bundle discovery, manifest parsing, digest calculation, and validation.
|
- `internal/bundle`: storage-backed source bundle discovery and validation using the public manifest contract.
|
||||||
- `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
|
- `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
|
||||||
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
|
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
|
||||||
- `internal/adapters/local`: local filesystem backend.
|
- `internal/adapters/local`: local filesystem backend.
|
||||||
@@ -18,15 +19,16 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
|
|||||||
- `internal/storage/fake`: in-memory backend for tests.
|
- `internal/storage/fake`: in-memory backend for tests.
|
||||||
- `internal/publish`: destination inspection, output planning, reconciliation, execution, managed cleanup, and explicit forced replacement.
|
- `internal/publish`: destination inspection, output planning, reconciliation, execution, managed cleanup, and explicit forced replacement.
|
||||||
- `internal/transform`: transform interface and registry.
|
- `internal/transform`: transform interface and registry.
|
||||||
- `internal/transform/markdown`: Markdown-to-HTML sidecar transform.
|
- `internal/transform/markdown`: Markdown-to-HTML transform.
|
||||||
- `internal/notify`: notification interface and current no-op notifier.
|
- `internal/notify`: notification interface and current no-op notifier.
|
||||||
- `internal/testutil`: shared test fixtures. Production code must not import this package.
|
- `internal/testutil`: shared test fixtures. Production code must not import this package.
|
||||||
- `docs`: current user, operator, policy, internal, and roadmap documentation.
|
- `docs`: current user, operator, policy, internal, and roadmap documentation.
|
||||||
- `examples`: copyable example configs and source bundles.
|
- `examples`: copyable example configs and source bundles.
|
||||||
|
|
||||||
Do not create new top-level package families such as `pkg`, `internal/stage`,
|
Do not create new top-level package families such as public `pkg/...` packages
|
||||||
`internal/modules`, or service-specific adapter directories unless the
|
beyond `pkg/bundle`, generic workflow containers, or service-specific adapter
|
||||||
architecture policy or a current roadmap explicitly calls for them.
|
directories unless the architecture policy or a current roadmap explicitly
|
||||||
|
calls for them.
|
||||||
|
|
||||||
## Common Commands
|
## Common Commands
|
||||||
|
|
||||||
@@ -73,7 +75,7 @@ GOCACHE=/private/tmp/distributor-gocache GOMODCACHE=/private/tmp/distributor-gom
|
|||||||
- Keep adapter packages thin. Backend-specific filesystem or service behavior belongs in adapters; bundle, state, transform, and publish policy belongs outside adapters.
|
- Keep adapter packages thin. Backend-specific filesystem or service behavior belongs in adapters; bundle, state, transform, and publish policy belongs outside adapters.
|
||||||
- Preserve public CLI behavior, config semantics, manifest schema, destination state schema, and implemented backend behavior unless the current task explicitly changes them.
|
- Preserve public CLI behavior, config semantics, manifest schema, destination state schema, and implemented backend behavior unless the current task explicitly changes them.
|
||||||
- Use `storage.DisplayPath`, `storage.StateFileName`, `storage.StatePath`, and `storage.ManagedBundleTargets` instead of duplicating those conventions.
|
- Use `storage.DisplayPath`, `storage.StateFileName`, `storage.StatePath`, and `storage.ManagedBundleTargets` instead of duplicating those conventions.
|
||||||
- Use `bundle.ValidateManifest` for normalized source manifest semantics, including embedded source manifests in destination state.
|
- Use `pkg/bundle` for normalized source manifest semantics. Internal packages should reach those rules through `internal/bundle` when they also need storage-backed bundle discovery or validation.
|
||||||
- Use `config.ValidatePublishTransformPolicy` for publish and transform policy combinations.
|
- Use `config.ValidatePublishTransformPolicy` for publish and transform policy combinations.
|
||||||
- Do not import concrete transform implementations from `internal/publish`; app-level wiring owns transform registration.
|
- Do not import concrete transform implementations from `internal/publish`; app-level wiring owns transform registration.
|
||||||
- Do not import `internal/testutil` from production code.
|
- Do not import `internal/testutil` from production code.
|
||||||
@@ -127,8 +129,9 @@ When adding or changing commands or flags:
|
|||||||
3. Add or update CLI tests in `internal/cli`.
|
3. Add or update CLI tests in `internal/cli`.
|
||||||
4. Update `docs/cli.md` if syntax, flags, output expectations, or workflows change.
|
4. Update `docs/cli.md` if syntax, flags, output expectations, or workflows change.
|
||||||
|
|
||||||
`validate` and `inspect` are local path commands. `run` loads configured
|
`validate` and `inspect` support a local path shortcut and configured
|
||||||
pipelines and executes local, SSH, and S3 backends.
|
source-only diagnostics. `run` loads configured pipelines and executes local,
|
||||||
|
SSH, and S3 backends.
|
||||||
|
|
||||||
## Storage Backends
|
## Storage Backends
|
||||||
|
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ Recommended:
|
|||||||
- `examples/`
|
- `examples/`
|
||||||
- `docs/policy/development.md`
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
### Modular, staged, service-oriented, or orchestration application
|
### Modular, service-oriented, or orchestration application
|
||||||
|
|
||||||
Required:
|
Required:
|
||||||
- `docs/cli.md`, if CLI-based
|
- `docs/cli.md`, if CLI-based
|
||||||
@@ -175,7 +175,7 @@ It should include:
|
|||||||
- dependency policy;
|
- dependency policy;
|
||||||
- how to add config fields;
|
- how to add config fields;
|
||||||
- how to add CLI flags;
|
- how to add CLI flags;
|
||||||
- how to add stages/modules/adapters, if applicable;
|
- how to add modules or adapters, if applicable;
|
||||||
- how to update examples;
|
- how to update examples;
|
||||||
- documentation update expectations.
|
- documentation update expectations.
|
||||||
|
|
||||||
@@ -216,7 +216,7 @@ Explain when commands are useful, not just their syntax.
|
|||||||
|
|
||||||
**Audience:** administrators, operators
|
**Audience:** administrators, operators
|
||||||
|
|
||||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||||
|
|
||||||
It should cover:
|
It should cover:
|
||||||
|
|
||||||
@@ -248,7 +248,7 @@ Each entry should include:
|
|||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
Required for modular, staged, service-oriented, or orchestration projects.
|
Required for modular, service-oriented, or orchestration projects.
|
||||||
|
|
||||||
This directory describes implemented internal components. It is not the roadmap.
|
This directory describes implemented internal components. It is not the roadmap.
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,8 @@
|
|||||||
# Implementation Roadmap
|
# Roadmap
|
||||||
|
|
||||||
This roadmap records current implementation status and deferred work for
|
This directory contains only future, deferred, or aspirational work for
|
||||||
`distributor`. Implemented behavior is documented in the user, operator, and
|
`distributor`. Implemented behavior is documented in the current user,
|
||||||
internal documentation listed below.
|
operator, internal, policy, integration, and example documentation:
|
||||||
|
|
||||||
Canonical current-behavior docs:
|
|
||||||
|
|
||||||
- `README.md`
|
- `README.md`
|
||||||
- `docs/cli.md`
|
- `docs/cli.md`
|
||||||
@@ -16,47 +14,89 @@ Canonical current-behavior docs:
|
|||||||
- `docs/policy/`
|
- `docs/policy/`
|
||||||
- `examples/`
|
- `examples/`
|
||||||
|
|
||||||
Future, planned, or aspirational behavior belongs under `docs/roadmap/` until
|
`distributor` currently supports local, SSH/SFTP, and S3-compatible source and
|
||||||
it is implemented.
|
destination backends; producer bundle creation through `pkg/bundle` and
|
||||||
|
`distributor manifest create`; configured source validation and inspection;
|
||||||
|
Markdown sidecar and `index.html` publication; archive and fixed destination
|
||||||
|
path mapping; destination link metadata; shared text/JSON CLI output; and
|
||||||
|
managed destination replacement behavior.
|
||||||
|
|
||||||
## Current State
|
## Future Work
|
||||||
|
|
||||||
`distributor` is ready for routine use against producer pipelines using the
|
These items are not implemented. They should not be documented as current
|
||||||
implemented local, SSH/SFTP, and S3-compatible backends.
|
behavior outside `docs/roadmap/` unless a future implementation adds them.
|
||||||
|
|
||||||
## Active Roadmap
|
### CLI And Status Output
|
||||||
|
|
||||||
There are no active implementation items in this roadmap.
|
- Add a root-global output flag only if the command parser is later refactored
|
||||||
|
around shared root options.
|
||||||
|
- Add output formats beyond `text` and `json` only if a concrete consumer
|
||||||
|
requires them.
|
||||||
|
- Add a versioned JSON schema reference after the first JSON-capable release.
|
||||||
|
- Add destination-state inspection behind an explicit flag such as
|
||||||
|
`--with-destinations` if operators need fan-out status diagnostics from
|
||||||
|
`inspect`.
|
||||||
|
- Add additional status or inspection presentation for destination primary
|
||||||
|
links beyond the current `run --format json` result model.
|
||||||
|
|
||||||
Before implementing new product behavior, add a focused roadmap entry when the
|
### Producer Workflows
|
||||||
work changes storage semantics, config, CLI behavior, state schema, transform
|
|
||||||
behavior, notification behavior, operational safety, or user-visible workflows.
|
|
||||||
Keep those entries out of current-behavior docs until the behavior exists.
|
|
||||||
|
|
||||||
## Deferred Work
|
- Add a no-write manifest creation mode, such as writing manifest JSON to
|
||||||
|
stdout, if producer pipelines need to capture manifests directly.
|
||||||
|
- Add broader producer workflow helpers, such as richer ignore rules or
|
||||||
|
template scaffolding, if real producer use cases require them.
|
||||||
|
- Add remote or storage-backed producer writers only if producer applications
|
||||||
|
need to assemble bundles outside the local filesystem.
|
||||||
|
|
||||||
These items are not implemented and should stay out of current-behavior docs
|
### Publication And Transform Behavior
|
||||||
until a roadmap entry is selected and implemented:
|
|
||||||
|
|
||||||
- external notification adapters;
|
- Add a separate collection or site-index transform if distributor needs
|
||||||
- warning-only digest mismatch handling;
|
multi-page aggregation.
|
||||||
- additional auth mechanisms beyond the implemented SSH and S3 credential
|
- Add richer transform metadata only if future state consumers need more than
|
||||||
paths;
|
the transform name and output path.
|
||||||
- compatibility parsing for legacy SSH URI config;
|
- Add custom HTML index output names only if fixed `index.html` is too limiting
|
||||||
- broad recursive destination deletion outside managed bundle paths;
|
for real deployments.
|
||||||
- concurrent fan-out publishing;
|
- Add richer fixed-destination source selection policies if deployments need
|
||||||
- streaming, resumable, or multipart S3 uploads;
|
something other than newest-by-`created`.
|
||||||
- cloud-provider-specific IAM integration docs;
|
- Add stricter handling for equal latest timestamps if timestamp ties become
|
||||||
- repository-managed packaging, release, and deployment automation.
|
common in producer workflows.
|
||||||
|
- Add higher-level status or approval workflows for fixed-root replacements if
|
||||||
|
dry-run output is not enough operational protection.
|
||||||
|
- Add richer link policies only if `auto`, `html`, and `source` prove
|
||||||
|
insufficient.
|
||||||
|
|
||||||
|
### State And Compatibility
|
||||||
|
|
||||||
|
- Define a post-release destination state schema bump policy before introducing
|
||||||
|
materially incompatible state changes.
|
||||||
|
- Add warning-only digest mismatch handling only if an operator workflow needs
|
||||||
|
publication to continue after validation failures.
|
||||||
|
- Add compatibility parsing for legacy SSH URI config only if migration support
|
||||||
|
is required.
|
||||||
|
|
||||||
|
### Backends, Security, And Deployment
|
||||||
|
|
||||||
|
- Add authentication mechanisms beyond the implemented SSH agent/key and S3
|
||||||
|
credential paths only when a concrete backend workflow requires them.
|
||||||
|
- Add broad recursive destination deletion outside managed bundle paths only if
|
||||||
|
a future design can preserve the current safety boundary.
|
||||||
|
- Add concurrent fan-out publishing only if runtime profiling shows it is
|
||||||
|
needed.
|
||||||
|
- Add streaming, resumable, or multipart S3 uploads only if object sizes make
|
||||||
|
the current write path insufficient.
|
||||||
|
- Add cloud-provider-specific IAM integration docs only when the repository
|
||||||
|
includes tested provider-specific behavior.
|
||||||
|
- Add repository-managed packaging, release, and deployment automation when the
|
||||||
|
release process is ready to be standardized.
|
||||||
|
|
||||||
## Roadmap Maintenance
|
## Roadmap Maintenance
|
||||||
|
|
||||||
When adding future roadmap work:
|
When adding future roadmap work:
|
||||||
|
|
||||||
- describe user-visible behavior and safety boundaries;
|
- describe user-visible behavior and safety boundaries;
|
||||||
- define which existing docs must change after implementation;
|
- define which current docs must change after implementation;
|
||||||
- keep examples secret-free and runnable or clearly environment-gated;
|
- keep examples secret-free and runnable or clearly environment-gated;
|
||||||
- avoid workflow labels in production code, tests, config fields, and user
|
- keep workflow labels out of production code, tests, config fields, and
|
||||||
documentation;
|
user-facing documentation;
|
||||||
- run focused tests for the changed behavior and `go test ./...` for
|
- run focused tests for the changed behavior and `go test ./...` for
|
||||||
cross-package changes.
|
cross-package changes.
|
||||||
|
|||||||
@@ -36,6 +36,67 @@ rg -n "backend:" <config-path>
|
|||||||
|
|
||||||
Safe fix: use `backend: local`, `backend: ssh`, or `backend: s3` for executable workflows.
|
Safe fix: use `backend: local`, `backend: ssh`, or `backend: s3` for executable workflows.
|
||||||
|
|
||||||
|
## `--format: format must be text or json`
|
||||||
|
|
||||||
|
Likely cause: a command was run with an unsupported output format.
|
||||||
|
|
||||||
|
Diagnostic:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --help
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix: use `--format text` or `--format json`. Help and usage output are always text.
|
||||||
|
|
||||||
|
## `--format json` wrote no JSON output
|
||||||
|
|
||||||
|
Likely cause: the command failed before it could construct a result, such as a missing config file, invalid arguments, unreadable secrets directory, or source setup failure.
|
||||||
|
|
||||||
|
Diagnostic:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config <config-path> --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix: read the stderr error and fix the setup problem. JSON mode writes a document only after the command has enough information to construct a result.
|
||||||
|
|
||||||
|
## `configured source mode requires --pipeline`
|
||||||
|
|
||||||
|
Likely cause: `validate` or `inspect` was run with `--config` but without an explicit pipeline id.
|
||||||
|
|
||||||
|
Diagnostic:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor validate --help
|
||||||
|
go run ./cmd/distributor inspect --help
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix: add `--pipeline <pipeline-id>`. Configured source diagnostics require an explicit pipeline even when the config contains one pipeline.
|
||||||
|
|
||||||
|
## `does not accept a local path with --config, --pipeline, or --bundle`
|
||||||
|
|
||||||
|
Likely cause: local-path mode and configured source mode were mixed in one `validate` or `inspect` command.
|
||||||
|
|
||||||
|
Diagnostic:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor inspect --help
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix: use either `distributor inspect <local-path>` or `distributor inspect --config <path> --pipeline <id>`, not both.
|
||||||
|
|
||||||
|
## `--format json` exited non-zero with `ok: false`
|
||||||
|
|
||||||
|
Likely cause: `run` began planning or executing destinations, and at least one destination failed while other destination results were still available.
|
||||||
|
|
||||||
|
Diagnostic:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config <config-path> --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix: inspect the top-level `errors` array, `result.actions`, and `result.summary`. Fix the failed destination, then preview with `--dry-run --format json` before retrying.
|
||||||
|
|
||||||
## `prefix must be a clean relative slash-separated path`
|
## `prefix must be a clean relative slash-separated path`
|
||||||
|
|
||||||
Likely cause: S3 `prefix` contains traversal, dot segments, empty segments, or backslashes after leading and trailing slashes are trimmed.
|
Likely cause: S3 `prefix` contains traversal, dot segments, empty segments, or backslashes after leading and trailing slashes are trimmed.
|
||||||
@@ -198,21 +259,33 @@ ssh-keygen -F <host> -f <known-hosts-path>
|
|||||||
|
|
||||||
Safe fix: verify the server identity out of band before updating `known_hosts`. Do not switch to `host_key_policy: off` to bypass an unexpected changed key.
|
Safe fix: verify the server identity out of band before updating `known_hosts`. Do not switch to `host_key_policy: off` to bypass an unexpected changed key.
|
||||||
|
|
||||||
## `stat ssh ... not_found` or `no bundles found`
|
## `pipeline "<id>" not found`
|
||||||
|
|
||||||
Likely cause: the configured SSH `path` is wrong, unreadable, or does not contain source bundles.
|
Likely cause: configured source validation or inspection requested a pipeline id that is not present in the config file.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
sftp <user>@<host>
|
rg -n "id:" <config-path>
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: correct the remote root `path`, permissions, or source bundle location.
|
Safe fix: pass an existing pipeline id with `--pipeline`, or update the config.
|
||||||
|
|
||||||
|
## `stat ssh ... not_found`, `stat s3 ... not_found`, or `no bundles found`
|
||||||
|
|
||||||
|
Likely cause: the configured source root is wrong, unreadable, or does not contain source bundles.
|
||||||
|
|
||||||
|
Diagnostic:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor validate --config <config-path> --pipeline <pipeline-id>
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix: correct the configured source root, S3 prefix, permissions, or source bundle location. Use `--bundle <path>` only with a source-root-relative bundle directory that contains `manifest.json`.
|
||||||
|
|
||||||
## `validate command requires a path` or `inspect command requires a path`
|
## `validate command requires a path` or `inspect command requires a path`
|
||||||
|
|
||||||
Likely cause: `validate` or `inspect` was run without a path.
|
Likely cause: `validate` or `inspect` was run without a local path and without configured source mode.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
@@ -221,7 +294,7 @@ go run ./cmd/distributor validate --help
|
|||||||
go run ./cmd/distributor inspect --help
|
go run ./cmd/distributor inspect --help
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: pass a local source bundle directory or a local tree containing source bundles.
|
Safe fix: pass a local source bundle directory or local tree, or pass both `--config <path>` and `--pipeline <id>`.
|
||||||
|
|
||||||
## `no bundles found under "."`
|
## `no bundles found under "."`
|
||||||
|
|
||||||
|
|||||||
29
examples/archive-and-latest.yml
Normal file
29
examples/archive-and-latest.yml
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
pipelines:
|
||||||
|
- id: example-archive-and-latest
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: examples/source-bundle
|
||||||
|
destinations:
|
||||||
|
- id: local-source-archive
|
||||||
|
backend: local
|
||||||
|
path: workspace/published/archive-and-latest/archive
|
||||||
|
path_mapping:
|
||||||
|
mode: preserve_relative
|
||||||
|
publish:
|
||||||
|
source: true
|
||||||
|
html: false
|
||||||
|
- id: local-html-latest
|
||||||
|
backend: local
|
||||||
|
path: workspace/published/archive-and-latest/latest
|
||||||
|
path_mapping:
|
||||||
|
mode: fixed
|
||||||
|
links:
|
||||||
|
base_url: https://reports.example.com/latest
|
||||||
|
primary: auto
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: index
|
||||||
16
examples/local-index.yml
Normal file
16
examples/local-index.yml
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
pipelines:
|
||||||
|
- id: example-index-bundle
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: examples/source-bundle
|
||||||
|
destinations:
|
||||||
|
- id: local-index
|
||||||
|
backend: local
|
||||||
|
path: workspace/published/index-bundle
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: index
|
||||||
@@ -161,31 +161,10 @@ func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOpti
|
|||||||
}
|
}
|
||||||
return b.translateError(storage.OpWalk, prefix, err)
|
return b.translateError(storage.OpWalk, prefix, err)
|
||||||
}
|
}
|
||||||
visited := 0
|
emitter := storage.NewWalkEmitter(ctx, backendName, opts, fn)
|
||||||
emit := func(entry storage.Entry) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if opts.Limit > 0 && visited >= opts.Limit {
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
}
|
|
||||||
visited++
|
|
||||||
if err := fn(entry); err != nil {
|
|
||||||
if errors.Is(err, storage.ErrStopWalk) {
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
}
|
|
||||||
return storage.NewError(storage.OpWalk, backendName, entry.Path, storage.ErrUnknown, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if !info.IsDir() {
|
if !info.IsDir() {
|
||||||
if err := emit(entryFromInfo(prefix, info)); errors.Is(err, storage.ErrStopWalk) {
|
return storage.FinishWalk(emitter.Emit(entryFromInfo(prefix, info)))
|
||||||
return nil
|
|
||||||
} else if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
walkErr := filepath.WalkDir(nativePrefix, func(nativePath string, dirEntry fs.DirEntry, err error) error {
|
walkErr := filepath.WalkDir(nativePrefix, func(nativePath string, dirEntry fs.DirEntry, err error) error {
|
||||||
@@ -210,24 +189,13 @@ func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOpti
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return b.translateError(storage.OpWalk, logicalPath, err)
|
return b.translateError(storage.OpWalk, logicalPath, err)
|
||||||
}
|
}
|
||||||
return emit(entryFromInfo(logicalPath, info))
|
return emitter.Emit(entryFromInfo(logicalPath, info))
|
||||||
})
|
})
|
||||||
if errors.Is(walkErr, storage.ErrStopWalk) {
|
return storage.FinishWalk(walkErr)
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return walkErr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||||
found := false
|
return storage.HasAny(ctx, b, prefix)
|
||||||
err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
|
|
||||||
found = true
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return found, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||||
|
|||||||
@@ -183,55 +183,25 @@ func (b *Backend) Walk(ctx context.Context, logicalPrefix string, opts storage.W
|
|||||||
if err := storage.ValidatePrefix(logicalPrefix); err != nil {
|
if err := storage.ValidatePrefix(logicalPrefix); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
visited := 0
|
emitter := storage.NewWalkEmitter(ctx, BackendName, opts, fn)
|
||||||
emit := func(entry storage.Entry) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if opts.Limit > 0 && visited >= opts.Limit {
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
}
|
|
||||||
visited++
|
|
||||||
if err := fn(entry); err != nil {
|
|
||||||
if errors.Is(err, storage.ErrStopWalk) {
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
}
|
|
||||||
return storage.NewError(storage.OpWalk, BackendName, entry.Path, storage.ErrUnknown, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if logicalPrefix != "" {
|
if logicalPrefix != "" {
|
||||||
entry, err := b.Stat(ctx, logicalPrefix)
|
entry, err := b.Stat(ctx, logicalPrefix)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if err := emit(entry); errors.Is(err, storage.ErrStopWalk) {
|
if err := emitter.Emit(entry); err != nil {
|
||||||
return nil
|
return storage.FinishWalk(err)
|
||||||
} else if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
if opts.Limit > 0 && visited >= opts.Limit {
|
if emitter.LimitReached() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
} else if !storage.IsNotFound(err) {
|
} else if !storage.IsNotFound(err) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
err := b.walkObjects(ctx, logicalPrefix, opts, emit)
|
return storage.FinishWalk(b.walkObjects(ctx, logicalPrefix, opts, emitter.Emit))
|
||||||
if errors.Is(err, storage.ErrStopWalk) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backend) HasAny(ctx context.Context, logicalPrefix string) (bool, error) {
|
func (b *Backend) HasAny(ctx context.Context, logicalPrefix string) (bool, error) {
|
||||||
found := false
|
return storage.HasAny(ctx, b, logicalPrefix)
|
||||||
err := b.Walk(ctx, logicalPrefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
|
|
||||||
found = true
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return found, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||||
|
|||||||
@@ -209,51 +209,17 @@ func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOpti
|
|||||||
return b.translateError(storage.OpWalk, prefix, err)
|
return b.translateError(storage.OpWalk, prefix, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
visited := 0
|
emitter := storage.NewWalkEmitter(ctx, BackendName, opts, fn)
|
||||||
emit := func(entry storage.Entry) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if opts.Limit > 0 && visited >= opts.Limit {
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
}
|
|
||||||
visited++
|
|
||||||
if err := fn(entry); err != nil {
|
|
||||||
if errors.Is(err, storage.ErrStopWalk) {
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
}
|
|
||||||
return storage.NewError(storage.OpWalk, BackendName, entry.Path, storage.ErrUnknown, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if !info.IsDir() {
|
if !info.IsDir() {
|
||||||
if err := emit(entryFromInfo(prefix, info)); errors.Is(err, storage.ErrStopWalk) {
|
return storage.FinishWalk(emitter.Emit(entryFromInfo(prefix, info)))
|
||||||
return nil
|
|
||||||
} else if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := b.walkDirectory(ctx, prefix, nativePrefix, opts, emit); errors.Is(err, storage.ErrStopWalk) {
|
return storage.FinishWalk(b.walkDirectory(ctx, prefix, nativePrefix, opts, emitter.Emit))
|
||||||
return nil
|
|
||||||
} else if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||||
found := false
|
return storage.HasAny(ctx, b, prefix)
|
||||||
err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
|
|
||||||
found = true
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return found, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||||
|
|||||||
@@ -37,6 +37,22 @@ type backendFactory struct {
|
|||||||
readOnlyKnownHosts bool
|
readOnlyKnownHosts bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type backendOpenSpec struct {
|
||||||
|
role string
|
||||||
|
backend string
|
||||||
|
path string
|
||||||
|
host string
|
||||||
|
user string
|
||||||
|
port int
|
||||||
|
ssh config.SSH
|
||||||
|
endpoint string
|
||||||
|
bucket string
|
||||||
|
prefix string
|
||||||
|
region string
|
||||||
|
forcePath *bool
|
||||||
|
credentials config.Credentials
|
||||||
|
}
|
||||||
|
|
||||||
func newBackendFactory() *backendFactory {
|
func newBackendFactory() *backendFactory {
|
||||||
return newBackendFactoryWithEnvironment(config.ProcessEnvironment())
|
return newBackendFactoryWithEnvironment(config.ProcessEnvironment())
|
||||||
}
|
}
|
||||||
@@ -91,25 +107,11 @@ func newBackendFactoryWithEnvironment(environment config.Environment) *backendFa
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) {
|
func (f *backendFactory) openSource(ctx context.Context, source config.Backend) (storage.Backend, error) {
|
||||||
if source.Backend != config.BackendLocal && source.Backend != config.BackendSSH && source.Backend != config.BackendS3 {
|
return f.openBackend(ctx, backendOpenSpecFromSource(source))
|
||||||
return nil, fmt.Errorf("source backend %s is not implemented for execution", source.Backend)
|
|
||||||
}
|
|
||||||
openConfig, err := f.sourceOpenConfig(source)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return f.registry.Open(ctx, source.Backend, openConfig)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *backendFactory) openDestination(ctx context.Context, destination config.Destination) (storage.Backend, error) {
|
func (f *backendFactory) openDestination(ctx context.Context, destination config.Destination) (storage.Backend, error) {
|
||||||
if destination.Backend != config.BackendLocal && destination.Backend != config.BackendSSH && destination.Backend != config.BackendS3 {
|
return f.openBackend(ctx, backendOpenSpecFromDestination(destination))
|
||||||
return nil, fmt.Errorf("backend %s is not implemented for execution", destination.Backend)
|
|
||||||
}
|
|
||||||
openConfig, err := f.destinationOpenConfig(destination)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return f.registry.Open(ctx, destination.Backend, openConfig)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storage.Backend, error) {
|
func (f *backendFactory) openLocalPath(ctx context.Context, path string) (storage.Backend, error) {
|
||||||
@@ -120,40 +122,51 @@ func (f *backendFactory) resolveCredentials(creds config.Credentials) (config.Re
|
|||||||
return f.environment.ResolveCredentials(creds)
|
return f.environment.ResolveCredentials(creds)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *backendFactory) sourceOpenConfig(source config.Backend) (storage.OpenConfig, error) {
|
func (f *backendFactory) openBackend(ctx context.Context, spec backendOpenSpec) (storage.Backend, error) {
|
||||||
cfg := sourceOpenConfig(source)
|
if !backendExecutable(spec.backend) {
|
||||||
if source.Backend == config.BackendS3 {
|
if spec.role == "source" {
|
||||||
if err := f.addS3Config(cfg, source.Endpoint, source.Bucket, source.Prefix, source.Region, source.ForcePath, source.Creds); err != nil {
|
return nil, fmt.Errorf("source backend %s is not implemented for execution", spec.backend)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("backend %s is not implemented for execution", spec.backend)
|
||||||
|
}
|
||||||
|
openConfig, err := f.openConfig(spec)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return f.registry.Open(ctx, spec.backend, openConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
func backendExecutable(name string) bool {
|
||||||
|
return name == config.BackendLocal || name == config.BackendSSH || name == config.BackendS3
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *backendFactory) openConfig(spec backendOpenSpec) (storage.OpenConfig, error) {
|
||||||
|
cfg := storage.OpenConfig{storagePathKey: spec.path}
|
||||||
|
switch spec.backend {
|
||||||
|
case config.BackendSSH:
|
||||||
|
cfg[sshHostKey] = spec.host
|
||||||
|
cfg[sshUserKey] = spec.user
|
||||||
|
cfg[sshPortKey] = strconv.Itoa(spec.port)
|
||||||
|
cfg[sshKeyFileKey] = spec.ssh.KeyFile
|
||||||
|
cfg[sshKnownHostsKey] = spec.ssh.KnownHosts
|
||||||
|
cfg[sshHostKeyPolicyKey] = string(spec.ssh.HostKeyPolicy)
|
||||||
|
cfg[sshReadOnlyHostsKey] = strconv.FormatBool(f.readOnlyKnownHosts)
|
||||||
|
case config.BackendS3:
|
||||||
|
if err := f.addS3Config(cfg, spec); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if source.Backend == config.BackendSSH {
|
|
||||||
cfg[sshReadOnlyHostsKey] = strconv.FormatBool(f.readOnlyKnownHosts)
|
|
||||||
}
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *backendFactory) destinationOpenConfig(destination config.Destination) (storage.OpenConfig, error) {
|
func (f *backendFactory) addS3Config(cfg storage.OpenConfig, spec backendOpenSpec) error {
|
||||||
cfg := destinationOpenConfig(destination)
|
cfg[s3EndpointKey] = spec.endpoint
|
||||||
if destination.Backend == config.BackendS3 {
|
cfg[s3BucketKey] = spec.bucket
|
||||||
if err := f.addS3Config(cfg, destination.Endpoint, destination.Bucket, destination.Prefix, destination.Region, destination.ForcePath, destination.Creds); err != nil {
|
cfg[s3PrefixKey] = spec.prefix
|
||||||
return nil, err
|
cfg[s3RegionKey] = spec.region
|
||||||
}
|
cfg[s3ForcePathStyleKey] = strconv.FormatBool(config.ForcePathStyle(spec.forcePath))
|
||||||
}
|
if spec.credentials.AccessKeyIDEnv != "" || spec.credentials.SecretAccessKeyEnv != "" {
|
||||||
if destination.Backend == config.BackendSSH {
|
resolved, err := f.resolveCredentials(spec.credentials)
|
||||||
cfg[sshReadOnlyHostsKey] = strconv.FormatBool(f.readOnlyKnownHosts)
|
|
||||||
}
|
|
||||||
return cfg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *backendFactory) addS3Config(cfg storage.OpenConfig, endpoint, bucket, prefix, region string, forcePath *bool, creds config.Credentials) error {
|
|
||||||
cfg[s3EndpointKey] = endpoint
|
|
||||||
cfg[s3BucketKey] = bucket
|
|
||||||
cfg[s3PrefixKey] = prefix
|
|
||||||
cfg[s3RegionKey] = region
|
|
||||||
cfg[s3ForcePathStyleKey] = strconv.FormatBool(config.ForcePathStyle(forcePath))
|
|
||||||
if creds.AccessKeyIDEnv != "" || creds.SecretAccessKeyEnv != "" {
|
|
||||||
resolved, err := f.resolveCredentials(creds)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -163,30 +176,38 @@ func (f *backendFactory) addS3Config(cfg storage.OpenConfig, endpoint, bucket, p
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func sourceOpenConfig(source config.Backend) storage.OpenConfig {
|
func backendOpenSpecFromSource(source config.Backend) backendOpenSpec {
|
||||||
cfg := storage.OpenConfig{storagePathKey: source.Path}
|
return backendOpenSpec{
|
||||||
if source.Backend == config.BackendSSH {
|
role: "source",
|
||||||
cfg[sshHostKey] = source.Host
|
backend: source.Backend,
|
||||||
cfg[sshUserKey] = source.User
|
path: source.Path,
|
||||||
cfg[sshPortKey] = strconv.Itoa(source.Port)
|
host: source.Host,
|
||||||
cfg[sshKeyFileKey] = source.SSH.KeyFile
|
user: source.User,
|
||||||
cfg[sshKnownHostsKey] = source.SSH.KnownHosts
|
port: source.Port,
|
||||||
cfg[sshHostKeyPolicyKey] = string(source.SSH.HostKeyPolicy)
|
ssh: source.SSH,
|
||||||
cfg[sshReadOnlyHostsKey] = "false"
|
endpoint: source.Endpoint,
|
||||||
|
bucket: source.Bucket,
|
||||||
|
prefix: source.Prefix,
|
||||||
|
region: source.Region,
|
||||||
|
forcePath: source.ForcePath,
|
||||||
|
credentials: source.Creds,
|
||||||
}
|
}
|
||||||
return cfg
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func destinationOpenConfig(destination config.Destination) storage.OpenConfig {
|
func backendOpenSpecFromDestination(destination config.Destination) backendOpenSpec {
|
||||||
cfg := storage.OpenConfig{storagePathKey: destination.Path}
|
return backendOpenSpec{
|
||||||
if destination.Backend == config.BackendSSH {
|
role: "destination",
|
||||||
cfg[sshHostKey] = destination.Host
|
backend: destination.Backend,
|
||||||
cfg[sshUserKey] = destination.User
|
path: destination.Path,
|
||||||
cfg[sshPortKey] = strconv.Itoa(destination.Port)
|
host: destination.Host,
|
||||||
cfg[sshKeyFileKey] = destination.SSH.KeyFile
|
user: destination.User,
|
||||||
cfg[sshKnownHostsKey] = destination.SSH.KnownHosts
|
port: destination.Port,
|
||||||
cfg[sshHostKeyPolicyKey] = string(destination.SSH.HostKeyPolicy)
|
ssh: destination.SSH,
|
||||||
cfg[sshReadOnlyHostsKey] = "false"
|
endpoint: destination.Endpoint,
|
||||||
|
bucket: destination.Bucket,
|
||||||
|
prefix: destination.Prefix,
|
||||||
|
region: destination.Region,
|
||||||
|
forcePath: destination.ForcePath,
|
||||||
|
credentials: destination.Creds,
|
||||||
}
|
}
|
||||||
return cfg
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,13 +1,16 @@
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBackendFactoryOpensLocalSource(t *testing.T) {
|
func TestBackendFactoryOpensLocalSource(t *testing.T) {
|
||||||
@@ -275,7 +278,8 @@ func TestBackendFactoryResolvesCredentialsThroughEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) {
|
func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) {
|
||||||
cfg := sourceOpenConfig(config.Backend{
|
factory := &backendFactory{environment: config.NewEnvironment(nil, nil)}
|
||||||
|
cfg, err := factory.openConfig(backendOpenSpecFromSource(config.Backend{
|
||||||
Backend: config.BackendSSH,
|
Backend: config.BackendSSH,
|
||||||
Host: "source.example.com",
|
Host: "source.example.com",
|
||||||
User: "reports",
|
User: "reports",
|
||||||
@@ -286,7 +290,10 @@ func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) {
|
|||||||
KnownHosts: "/home/reports/.ssh/known_hosts",
|
KnownHosts: "/home/reports/.ssh/known_hosts",
|
||||||
HostKeyPolicy: config.HostKeyPolicyStrict,
|
HostKeyPolicy: config.HostKeyPolicyStrict,
|
||||||
},
|
},
|
||||||
})
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("openConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
assertOpenConfig(t, cfg, map[string]string{
|
assertOpenConfig(t, cfg, map[string]string{
|
||||||
storagePathKey: "/reports",
|
storagePathKey: "/reports",
|
||||||
@@ -300,7 +307,8 @@ func TestBackendFactoryBuildsSSHSourceOpenConfig(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBackendFactoryBuildsSSHDestinationOpenConfig(t *testing.T) {
|
func TestBackendFactoryBuildsSSHDestinationOpenConfig(t *testing.T) {
|
||||||
cfg := destinationOpenConfig(config.Destination{
|
factory := &backendFactory{environment: config.NewEnvironment(nil, nil)}
|
||||||
|
cfg, err := factory.openConfig(backendOpenSpecFromDestination(config.Destination{
|
||||||
Backend: config.BackendSSH,
|
Backend: config.BackendSSH,
|
||||||
Host: "destination.example.com",
|
Host: "destination.example.com",
|
||||||
User: "deploy",
|
User: "deploy",
|
||||||
@@ -309,7 +317,10 @@ func TestBackendFactoryBuildsSSHDestinationOpenConfig(t *testing.T) {
|
|||||||
SSH: config.SSH{
|
SSH: config.SSH{
|
||||||
HostKeyPolicy: config.HostKeyPolicyAcceptNew,
|
HostKeyPolicy: config.HostKeyPolicyAcceptNew,
|
||||||
},
|
},
|
||||||
})
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("openConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
assertOpenConfig(t, cfg, map[string]string{
|
assertOpenConfig(t, cfg, map[string]string{
|
||||||
storagePathKey: "/srv/archive",
|
storagePathKey: "/srv/archive",
|
||||||
@@ -320,6 +331,257 @@ func TestBackendFactoryBuildsSSHDestinationOpenConfig(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBackendFactoryBuildsEquivalentSourceAndDestinationOpenConfig(t *testing.T) {
|
||||||
|
forcePathStyle := false
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
source config.Backend
|
||||||
|
destination config.Destination
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "local",
|
||||||
|
source: config.Backend{Backend: config.BackendLocal, Path: "/reports"},
|
||||||
|
destination: config.Destination{Backend: config.BackendLocal, Path: "/reports"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ssh",
|
||||||
|
source: config.Backend{
|
||||||
|
Backend: config.BackendSSH,
|
||||||
|
Host: "reports.example.com",
|
||||||
|
User: "reports",
|
||||||
|
Port: 2222,
|
||||||
|
Path: "/reports",
|
||||||
|
SSH: config.SSH{
|
||||||
|
KeyFile: "/home/reports/.ssh/id_ed25519",
|
||||||
|
KnownHosts: "/home/reports/.ssh/known_hosts",
|
||||||
|
HostKeyPolicy: config.HostKeyPolicyStrict,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
destination: config.Destination{
|
||||||
|
Backend: config.BackendSSH,
|
||||||
|
Host: "reports.example.com",
|
||||||
|
User: "reports",
|
||||||
|
Port: 2222,
|
||||||
|
Path: "/reports",
|
||||||
|
SSH: config.SSH{
|
||||||
|
KeyFile: "/home/reports/.ssh/id_ed25519",
|
||||||
|
KnownHosts: "/home/reports/.ssh/known_hosts",
|
||||||
|
HostKeyPolicy: config.HostKeyPolicyStrict,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "s3",
|
||||||
|
source: config.Backend{
|
||||||
|
Backend: config.BackendS3,
|
||||||
|
Endpoint: "https://s3.example.com",
|
||||||
|
Bucket: "reports",
|
||||||
|
Prefix: "archive",
|
||||||
|
Region: "us-west-2",
|
||||||
|
ForcePath: &forcePathStyle,
|
||||||
|
},
|
||||||
|
destination: config.Destination{
|
||||||
|
Backend: config.BackendS3,
|
||||||
|
Endpoint: "https://s3.example.com",
|
||||||
|
Bucket: "reports",
|
||||||
|
Prefix: "archive",
|
||||||
|
Region: "us-west-2",
|
||||||
|
ForcePath: &forcePathStyle,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "s3 explicit credentials",
|
||||||
|
source: config.Backend{
|
||||||
|
Backend: config.BackendS3,
|
||||||
|
Endpoint: "https://s3.example.com",
|
||||||
|
Bucket: "reports",
|
||||||
|
Region: config.DefaultS3Region,
|
||||||
|
Creds: config.Credentials{
|
||||||
|
AccessKeyIDEnv: "ACCESS_KEY_ID",
|
||||||
|
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
destination: config.Destination{
|
||||||
|
Backend: config.BackendS3,
|
||||||
|
Endpoint: "https://s3.example.com",
|
||||||
|
Bucket: "reports",
|
||||||
|
Region: config.DefaultS3Region,
|
||||||
|
Creds: config.Credentials{
|
||||||
|
AccessKeyIDEnv: "ACCESS_KEY_ID",
|
||||||
|
SecretAccessKeyEnv: "SECRET_ACCESS_KEY",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
factory := &backendFactory{
|
||||||
|
environment: config.NewEnvironment(map[string]string{
|
||||||
|
"ACCESS_KEY_ID": "secret-access",
|
||||||
|
"SECRET_ACCESS_KEY": "secret-secret",
|
||||||
|
}, func(string) (string, bool) { return "", false }),
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
sourceConfig, err := factory.openConfig(backendOpenSpecFromSource(tt.source))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("source openConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
destinationConfig, err := factory.openConfig(backendOpenSpecFromDestination(tt.destination))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("destination openConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if !openConfigEqual(sourceConfig, destinationConfig) {
|
||||||
|
t.Fatalf("source open config = %#v, destination open config = %#v, want equivalent", sourceConfig, destinationConfig)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendFactoryBuildsEquivalentDryRunSSHOpenConfig(t *testing.T) {
|
||||||
|
factory := &backendFactory{readOnlyKnownHosts: true}
|
||||||
|
sourceConfig, err := factory.openConfig(backendOpenSpecFromSource(config.Backend{
|
||||||
|
Backend: config.BackendSSH,
|
||||||
|
Host: "reports.example.com",
|
||||||
|
Port: 22,
|
||||||
|
Path: "/reports",
|
||||||
|
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("source openConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
destinationConfig, err := factory.openConfig(backendOpenSpecFromDestination(config.Destination{
|
||||||
|
Backend: config.BackendSSH,
|
||||||
|
Host: "reports.example.com",
|
||||||
|
Port: 22,
|
||||||
|
Path: "/reports",
|
||||||
|
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyAcceptNew},
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("destination openConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if !openConfigEqual(sourceConfig, destinationConfig) {
|
||||||
|
t.Fatalf("source open config = %#v, destination open config = %#v, want equivalent", sourceConfig, destinationConfig)
|
||||||
|
}
|
||||||
|
if sourceConfig[sshReadOnlyHostsKey] != "true" {
|
||||||
|
t.Fatalf("open config %s = %q, want true", sshReadOnlyHostsKey, sourceConfig[sshReadOnlyHostsKey])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfiguredSourceValidationAndRunUseEquivalentSourceOpenConfig(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
source config.Backend
|
||||||
|
sourceKey string
|
||||||
|
dest config.Destination
|
||||||
|
destKey string
|
||||||
|
wantFields map[string]string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "s3",
|
||||||
|
source: config.Backend{
|
||||||
|
Backend: config.BackendS3,
|
||||||
|
Endpoint: "https://s3.example.com",
|
||||||
|
Bucket: "source-bucket",
|
||||||
|
Prefix: "source-prefix",
|
||||||
|
Region: config.DefaultS3Region,
|
||||||
|
},
|
||||||
|
sourceKey: "s3:source-bucket",
|
||||||
|
dest: config.Destination{
|
||||||
|
ID: "archive",
|
||||||
|
Backend: config.BackendS3,
|
||||||
|
Endpoint: "https://s3.example.com",
|
||||||
|
Bucket: "destination-bucket",
|
||||||
|
Region: config.DefaultS3Region,
|
||||||
|
},
|
||||||
|
destKey: "s3:destination-bucket",
|
||||||
|
wantFields: map[string]string{
|
||||||
|
s3EndpointKey: "https://s3.example.com",
|
||||||
|
s3BucketKey: "source-bucket",
|
||||||
|
s3PrefixKey: "source-prefix",
|
||||||
|
s3RegionKey: config.DefaultS3Region,
|
||||||
|
s3ForcePathStyleKey: "true",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ssh",
|
||||||
|
source: config.Backend{
|
||||||
|
Backend: config.BackendSSH,
|
||||||
|
Host: "ssh.example.com",
|
||||||
|
User: "reports",
|
||||||
|
Port: 2222,
|
||||||
|
Path: "/source",
|
||||||
|
SSH: config.SSH{
|
||||||
|
KeyFile: "/home/reports/.ssh/id_ed25519",
|
||||||
|
KnownHosts: "/home/reports/.ssh/known_hosts",
|
||||||
|
HostKeyPolicy: config.HostKeyPolicyStrict,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sourceKey: "ssh:/source",
|
||||||
|
dest: config.Destination{
|
||||||
|
ID: "archive",
|
||||||
|
Backend: config.BackendSSH,
|
||||||
|
Host: "ssh.example.com",
|
||||||
|
Port: 2222,
|
||||||
|
Path: "/destination",
|
||||||
|
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyStrict},
|
||||||
|
},
|
||||||
|
destKey: "ssh:/destination",
|
||||||
|
wantFields: map[string]string{
|
||||||
|
storagePathKey: "/source",
|
||||||
|
sshHostKey: "ssh.example.com",
|
||||||
|
sshUserKey: "reports",
|
||||||
|
sshPortKey: "2222",
|
||||||
|
sshKeyFileKey: "/home/reports/.ssh/id_ed25519",
|
||||||
|
sshKnownHostsKey: "/home/reports/.ssh/known_hosts",
|
||||||
|
sshHostKeyPolicyKey: "strict",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
sourceBackend := fake.New()
|
||||||
|
testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{ID: "reports.source"})
|
||||||
|
destinationBackend := fake.New()
|
||||||
|
var validateSourceConfig storage.OpenConfig
|
||||||
|
var runSourceConfig storage.OpenConfig
|
||||||
|
validateProvider := recordingBackendFactoryProvider(t, map[string]storage.Backend{
|
||||||
|
tt.sourceKey: sourceBackend,
|
||||||
|
tt.destKey: destinationBackend,
|
||||||
|
}, func(cfg storage.OpenConfig) {
|
||||||
|
validateSourceConfig = cfg
|
||||||
|
})
|
||||||
|
runProvider := recordingBackendFactoryProvider(t, map[string]storage.Backend{
|
||||||
|
tt.sourceKey: sourceBackend,
|
||||||
|
tt.destKey: destinationBackend,
|
||||||
|
}, func(cfg storage.OpenConfig) {
|
||||||
|
runSourceConfig = cfg
|
||||||
|
})
|
||||||
|
cfg := config.Config{Pipelines: []config.Pipeline{{
|
||||||
|
ID: "reports",
|
||||||
|
Source: tt.source,
|
||||||
|
Destinations: []config.Destination{tt.dest},
|
||||||
|
}}}
|
||||||
|
config.ApplyDefaults(&cfg)
|
||||||
|
|
||||||
|
var validateOutput bytes.Buffer
|
||||||
|
if err := validateConfigWithBackendFactory(context.Background(), cfg, ValidateOptions{
|
||||||
|
PipelineID: "reports",
|
||||||
|
Stdout: &validateOutput,
|
||||||
|
}, validateProvider); err != nil {
|
||||||
|
t.Fatalf("validateConfigWithBackendFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{}, runProvider); err != nil {
|
||||||
|
t.Fatalf("runConfigWithBackendFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
if !openConfigEqual(validateSourceConfig, runSourceConfig) {
|
||||||
|
t.Fatalf("validate source config = %#v, run source config = %#v, want equivalent", validateSourceConfig, runSourceConfig)
|
||||||
|
}
|
||||||
|
assertOpenConfig(t, runSourceConfig, tt.wantFields)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func assertOpenConfig(t *testing.T, got map[string]string, want map[string]string) {
|
func assertOpenConfig(t *testing.T, got map[string]string, want map[string]string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
for key, wantValue := range want {
|
for key, wantValue := range want {
|
||||||
@@ -328,3 +590,49 @@ func assertOpenConfig(t *testing.T, got map[string]string, want map[string]strin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func openConfigEqual(left, right storage.OpenConfig) bool {
|
||||||
|
if len(left) != len(right) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for key, leftValue := range left {
|
||||||
|
if right[key] != leftValue {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordingBackendFactoryProvider(t *testing.T, remoteBackends map[string]storage.Backend, recordSource func(storage.OpenConfig)) backendFactoryProvider {
|
||||||
|
t.Helper()
|
||||||
|
return func(environment config.Environment) *backendFactory {
|
||||||
|
registry := storage.NewRegistry()
|
||||||
|
if err := registry.Register(config.BackendS3, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||||
|
if cfg[s3BucketKey] == "source-bucket" {
|
||||||
|
recordSource(cfg)
|
||||||
|
}
|
||||||
|
key := "s3:" + cfg[s3BucketKey]
|
||||||
|
backend := remoteBackends[key]
|
||||||
|
if backend == nil {
|
||||||
|
return nil, fmt.Errorf("missing fake backend for %s", key)
|
||||||
|
}
|
||||||
|
return backend, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register s3 backend: %v", err)
|
||||||
|
}
|
||||||
|
if err := registry.Register(config.BackendSSH, func(ctx context.Context, cfg storage.OpenConfig) (storage.Backend, error) {
|
||||||
|
if cfg[storagePathKey] == "/source" {
|
||||||
|
recordSource(cfg)
|
||||||
|
}
|
||||||
|
key := "ssh:" + cfg[storagePathKey]
|
||||||
|
backend := remoteBackends[key]
|
||||||
|
if backend == nil {
|
||||||
|
return nil, fmt.Errorf("missing fake backend for %s", key)
|
||||||
|
}
|
||||||
|
return backend, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register ssh backend: %v", err)
|
||||||
|
}
|
||||||
|
return &backendFactory{registry: registry, environment: environment}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,38 +5,130 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
type InspectOptions struct {
|
type InspectOptions struct {
|
||||||
Path string
|
Path string
|
||||||
Stdout io.Writer
|
ConfigPath string
|
||||||
|
PipelineID string
|
||||||
|
BundlePath string
|
||||||
|
Stdout io.Writer
|
||||||
|
OutputFormat OutputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
func Inspect(ctx context.Context, options InspectOptions) error {
|
func Inspect(ctx context.Context, options InspectOptions) error {
|
||||||
if options.Path == "" {
|
return inspectWithBackendFactory(ctx, options, newBackendFactoryWithEnvironment)
|
||||||
return fmt.Errorf("inspect command requires a path")
|
|
||||||
}
|
|
||||||
backend, err := newBackendFactory().openLocalPath(ctx, options.Path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
bundles, err := bundle.Discover(ctx, backend, "")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return writeInspection(options.Stdout, bundles)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeInspection(w io.Writer, bundles []bundle.Bundle) error {
|
func inspectWithBackendFactory(ctx context.Context, options InspectOptions, provider backendFactoryProvider) error {
|
||||||
|
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
selection, err := selectSourceBundles(ctx, sourceCommandOptions{
|
||||||
|
CommandName: "inspect",
|
||||||
|
Path: options.Path,
|
||||||
|
ConfigPath: options.ConfigPath,
|
||||||
|
PipelineID: options.PipelineID,
|
||||||
|
BundlePath: options.BundlePath,
|
||||||
|
}, provider)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return writeInspectResult(options, selection)
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectConfigWithBackendFactory(ctx context.Context, cfg config.Config, options InspectOptions, provider backendFactoryProvider) error {
|
||||||
|
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
selection, err := selectSourceBundlesFromConfig(ctx, cfg, sourceCommandOptions{
|
||||||
|
CommandName: "inspect",
|
||||||
|
PipelineID: options.PipelineID,
|
||||||
|
BundlePath: options.BundlePath,
|
||||||
|
}, provider)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return writeInspectResult(options, selection)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeInspectResult(options InspectOptions, selection sourceSelection) error {
|
||||||
|
if IsJSONOutput(options.OutputFormat) {
|
||||||
|
return WriteJSONEnvelope(options.Stdout, "inspect", true, selection.Warnings, inspectResultFromSelection(selection), nil)
|
||||||
|
}
|
||||||
|
if err := writeWarnings(options.Stdout, selection.Warnings); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return writeInspection(options.Stdout, selection)
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectResult struct {
|
||||||
|
PipelineID string `json:"pipeline_id,omitempty"`
|
||||||
|
SourceBackend string `json:"source_backend,omitempty"`
|
||||||
|
BundleCount int `json:"bundle_count"`
|
||||||
|
Bundles []inspectBundleResult `json:"bundles"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectBundleResult struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Created string `json:"created"`
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
FileCount int `json:"file_count"`
|
||||||
|
TotalSize int64 `json:"total_size"`
|
||||||
|
Files []inspectFileResult `json:"files"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectFileResult struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectResultFromSelection(selection sourceSelection) inspectResult {
|
||||||
|
result := inspectResult{
|
||||||
|
PipelineID: selection.PipelineID,
|
||||||
|
SourceBackend: selection.SourceBackend,
|
||||||
|
BundleCount: len(selection.Bundles),
|
||||||
|
Bundles: make([]inspectBundleResult, 0, len(selection.Bundles)),
|
||||||
|
}
|
||||||
|
for _, sourceBundle := range selection.Bundles {
|
||||||
|
bundleResult := inspectBundleResult{
|
||||||
|
Path: storage.DisplayPath(sourceBundle.RootRelativePath),
|
||||||
|
ID: sourceBundle.Manifest.ID,
|
||||||
|
Created: sourceBundle.Manifest.Created.Format("2006-01-02T15:04:05Z07:00"),
|
||||||
|
Digest: sourceBundle.Manifest.Digest,
|
||||||
|
FileCount: len(sourceBundle.Manifest.Files),
|
||||||
|
Files: make([]inspectFileResult, 0, len(sourceBundle.Manifest.Files)),
|
||||||
|
}
|
||||||
|
for _, file := range sourceBundle.Manifest.Files {
|
||||||
|
bundleResult.TotalSize += file.Size
|
||||||
|
bundleResult.Files = append(bundleResult.Files, inspectFileResult{
|
||||||
|
Path: file.Path,
|
||||||
|
SHA256: file.SHA256,
|
||||||
|
Size: file.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
result.Bundles = append(result.Bundles, bundleResult)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeInspection(w io.Writer, selection sourceSelection) error {
|
||||||
if w == nil {
|
if w == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if _, err := fmt.Fprintf(w, "Bundles: %d\n", len(bundles)); err != nil {
|
if selection.ConfigMode {
|
||||||
|
if _, err := fmt.Fprintf(w, "Pipeline: %s\nSource: %s\n", selection.PipelineID, selection.SourceBackend); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := fmt.Fprintf(w, "Bundles: %d\n", len(selection.Bundles)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, sourceBundle := range bundles {
|
for _, sourceBundle := range selection.Bundles {
|
||||||
if _, err := fmt.Fprintf(
|
if _, err := fmt.Fprintf(
|
||||||
w,
|
w,
|
||||||
"- path=%s id=%s created=%s digest=%s files=%d\n",
|
"- path=%s id=%s created=%s digest=%s files=%d\n",
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestInspectPrintsBundleSummary(t *testing.T) {
|
func TestInspectPrintsBundleSummary(t *testing.T) {
|
||||||
@@ -32,6 +34,57 @@ func TestInspectPrintsBundleSummary(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInspectConfiguredLocalSource(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "daily", testutil.BundleOptions{ID: "reports.daily"})
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
err := Inspect(context.Background(), InspectOptions{
|
||||||
|
ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
PipelineID: "reports",
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Inspect() configured source error = %v", err)
|
||||||
|
}
|
||||||
|
output := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"Pipeline: reports",
|
||||||
|
"Source: local",
|
||||||
|
"Bundles: 1",
|
||||||
|
"path=daily",
|
||||||
|
"id=reports.daily",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(output, want) {
|
||||||
|
t.Fatalf("Inspect() output = %q, want substring %q", output, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectConfiguredSourceJSON(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{ID: "reports.json"})
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
err := Inspect(context.Background(), InspectOptions{
|
||||||
|
ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
PipelineID: "reports",
|
||||||
|
Stdout: &stdout,
|
||||||
|
OutputFormat: OutputFormatJSON,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Inspect() configured JSON error = %v", err)
|
||||||
|
}
|
||||||
|
result := decodeAppResult(t, stdout.String())
|
||||||
|
if result["pipeline_id"] != "reports" || result["source_backend"] != "local" || result["bundle_count"] != float64(1) {
|
||||||
|
t.Fatalf("result = %#v, want configured inspect metadata", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestInspectRequiresPath(t *testing.T) {
|
func TestInspectRequiresPath(t *testing.T) {
|
||||||
err := Inspect(context.Background(), InspectOptions{})
|
err := Inspect(context.Background(), InspectOptions{})
|
||||||
if err == nil || !strings.Contains(err.Error(), "requires a path") {
|
if err == nil || !strings.Contains(err.Error(), "requires a path") {
|
||||||
|
|||||||
128
internal/app/manifest.go
Normal file
128
internal/app/manifest.go
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ManifestCreateOptions struct {
|
||||||
|
Root string
|
||||||
|
ID string
|
||||||
|
Created string
|
||||||
|
Files []string
|
||||||
|
Overwrite bool
|
||||||
|
Stdout io.Writer
|
||||||
|
OutputFormat OutputFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
func ManifestCreate(ctx context.Context, options ManifestCreateOptions) error {
|
||||||
|
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if options.Root == "" {
|
||||||
|
return fmt.Errorf("manifest create command requires a bundle path")
|
||||||
|
}
|
||||||
|
if options.ID == "" {
|
||||||
|
return fmt.Errorf("manifest create command requires --id")
|
||||||
|
}
|
||||||
|
|
||||||
|
created, err := parseOptionalCreated(options.Created)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
files := normalizeManifestFiles(options.Files)
|
||||||
|
buildOptions := producerbundle.BuildOptions{
|
||||||
|
Root: options.Root,
|
||||||
|
ID: options.ID,
|
||||||
|
Created: created,
|
||||||
|
Files: files,
|
||||||
|
Scan: len(files) == 0,
|
||||||
|
}
|
||||||
|
manifest, err := producerbundle.BuildManifest(buildOptions)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := producerbundle.WriteManifest(options.Root, manifest, producerbundle.WriteManifestOptions{Overwrite: options.Overwrite}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
loaded, err := producerbundle.LoadManifest(options.Root)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := producerbundle.ValidateBundle(options.Root, loaded); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
result := manifestCreateResultFromManifest(options.Root, loaded)
|
||||||
|
if IsJSONOutput(options.OutputFormat) {
|
||||||
|
return WriteJSONEnvelope(options.Stdout, "manifest create", true, nil, result, nil)
|
||||||
|
}
|
||||||
|
if options.Stdout != nil {
|
||||||
|
_, err = fmt.Fprintf(options.Stdout, "created %s\nbundle: %s\nfiles: %d\ndigest: %s\n", producerbundle.ManifestName, result.ID, result.FileCount, result.Digest)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseOptionalCreated(value string) (time.Time, error) {
|
||||||
|
if value == "" {
|
||||||
|
return time.Time{}, nil
|
||||||
|
}
|
||||||
|
created, err := time.Parse(time.RFC3339, value)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("created must be RFC3339: %w", err)
|
||||||
|
}
|
||||||
|
return created, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeManifestFiles(files []string) []string {
|
||||||
|
normalized := make([]string, 0, len(files))
|
||||||
|
for _, file := range files {
|
||||||
|
normalized = append(normalized, filepath.ToSlash(filepath.Clean(strings.ReplaceAll(file, "\\", string(filepath.Separator)))))
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
type manifestCreateResult struct {
|
||||||
|
ManifestPath string `json:"manifest_path"`
|
||||||
|
Root string `json:"root"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Created string `json:"created"`
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
FileCount int `json:"file_count"`
|
||||||
|
Files []manifestCreateFileResult `json:"files"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type manifestCreateFileResult struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestCreateResultFromManifest(root string, manifest producerbundle.Manifest) manifestCreateResult {
|
||||||
|
result := manifestCreateResult{
|
||||||
|
ManifestPath: filepath.ToSlash(filepath.Join(root, producerbundle.ManifestName)),
|
||||||
|
Root: filepath.ToSlash(root),
|
||||||
|
ID: manifest.ID,
|
||||||
|
Created: manifest.Created.Format(time.RFC3339),
|
||||||
|
Digest: manifest.Digest,
|
||||||
|
FileCount: len(manifest.Files),
|
||||||
|
Files: make([]manifestCreateFileResult, 0, len(manifest.Files)),
|
||||||
|
}
|
||||||
|
for _, file := range manifest.Files {
|
||||||
|
result.Files = append(result.Files, manifestCreateFileResult{
|
||||||
|
Path: file.Path,
|
||||||
|
SHA256: file.SHA256,
|
||||||
|
Size: file.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
76
internal/app/output.go
Normal file
76
internal/app/output.go
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
const outputSchemaVersion = 1
|
||||||
|
|
||||||
|
type OutputFormat string
|
||||||
|
|
||||||
|
const (
|
||||||
|
OutputFormatText OutputFormat = "text"
|
||||||
|
OutputFormatJSON OutputFormat = "json"
|
||||||
|
)
|
||||||
|
|
||||||
|
type OutputWarning struct {
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type OutputError struct {
|
||||||
|
PipelineID string `json:"pipeline_id,omitempty"`
|
||||||
|
DestinationID string `json:"destination_id,omitempty"`
|
||||||
|
Backend string `json:"backend,omitempty"`
|
||||||
|
BundlePath string `json:"bundle_path,omitempty"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type outputEnvelope struct {
|
||||||
|
SchemaVersion int `json:"schema_version"`
|
||||||
|
Command string `json:"command"`
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Warnings []OutputWarning `json:"warnings"`
|
||||||
|
Result any `json:"result"`
|
||||||
|
Errors []OutputError `json:"errors,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func NormalizeOutputFormat(format OutputFormat) OutputFormat {
|
||||||
|
if format == "" {
|
||||||
|
return OutputFormatText
|
||||||
|
}
|
||||||
|
return format
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidateOutputFormat(format OutputFormat) error {
|
||||||
|
switch NormalizeOutputFormat(format) {
|
||||||
|
case OutputFormatText, OutputFormatJSON:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("format must be text or json")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsJSONOutput(format OutputFormat) bool {
|
||||||
|
return NormalizeOutputFormat(format) == OutputFormatJSON
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteJSONEnvelope(w io.Writer, command string, ok bool, warnings []OutputWarning, result any, errors []OutputError) error {
|
||||||
|
if w == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if warnings == nil {
|
||||||
|
warnings = []OutputWarning{}
|
||||||
|
}
|
||||||
|
envelope := outputEnvelope{
|
||||||
|
SchemaVersion: outputSchemaVersion,
|
||||||
|
Command: command,
|
||||||
|
OK: ok,
|
||||||
|
Warnings: warnings,
|
||||||
|
Result: result,
|
||||||
|
Errors: errors,
|
||||||
|
}
|
||||||
|
encoder := json.NewEncoder(w)
|
||||||
|
return encoder.Encode(envelope)
|
||||||
|
}
|
||||||
@@ -2,10 +2,8 @@ package app
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
@@ -15,14 +13,18 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type RunOptions struct {
|
type RunOptions struct {
|
||||||
ConfigPath string
|
ConfigPath string
|
||||||
DryRun bool
|
DryRun bool
|
||||||
Force bool
|
Force bool
|
||||||
Stdout io.Writer
|
Stdout io.Writer
|
||||||
Notifier notify.Notifier
|
OutputFormat OutputFormat
|
||||||
|
Notifier notify.Notifier
|
||||||
}
|
}
|
||||||
|
|
||||||
func Run(ctx context.Context, options RunOptions) error {
|
func Run(ctx context.Context, options RunOptions) error {
|
||||||
|
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -49,28 +51,41 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
|
|||||||
if notifier == nil {
|
if notifier == nil {
|
||||||
notifier = notify.Noop{}
|
notifier = notify.Noop{}
|
||||||
}
|
}
|
||||||
|
jsonOutput := IsJSONOutput(options.OutputFormat)
|
||||||
summary := runSummary{dryRun: options.DryRun}
|
summary := runSummary{dryRun: options.DryRun}
|
||||||
|
result := runResult{
|
||||||
|
DryRun: options.DryRun,
|
||||||
|
Pipelines: []runPipelineResult{},
|
||||||
|
Actions: []runActionResult{},
|
||||||
|
}
|
||||||
|
var warnings []OutputWarning
|
||||||
var failures runFailures
|
var failures runFailures
|
||||||
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
|
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if options.Stdout != nil {
|
secretWarnings := secretConflictWarnings(secretLoad.Conflicts)
|
||||||
if err := writeSecretConflictWarnings(options.Stdout, secretLoad.Conflicts); err != nil {
|
if jsonOutput {
|
||||||
|
warnings = append(warnings, secretWarnings...)
|
||||||
|
} else if options.Stdout != nil {
|
||||||
|
if err := writeWarnings(options.Stdout, secretWarnings); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
backends := provider(secretLoad.Environment)
|
backends := provider(secretLoad.Environment)
|
||||||
backends.readOnlyKnownHosts = options.DryRun
|
backends.readOnlyKnownHosts = options.DryRun
|
||||||
transforms := newTransformRegistry()
|
transforms := newTransformRegistry()
|
||||||
if options.Stdout != nil {
|
if options.Stdout != nil && !jsonOutput {
|
||||||
if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
|
if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, pipeline := range cfg.Pipelines {
|
for _, pipeline := range cfg.Pipelines {
|
||||||
if options.Stdout != nil {
|
pipelineWarnings := sshWarnings(pipeline)
|
||||||
if err := writeSSHWarnings(options.Stdout, pipeline); err != nil {
|
if jsonOutput {
|
||||||
|
warnings = append(warnings, pipelineWarnings...)
|
||||||
|
} else if options.Stdout != nil {
|
||||||
|
if err := writeWarnings(options.Stdout, pipelineWarnings); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -83,53 +98,114 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
|
|||||||
closeBackend(sourceBackend)
|
closeBackend(sourceBackend)
|
||||||
return fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
|
return fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||||
}
|
}
|
||||||
if options.Stdout != nil {
|
result.Pipelines = append(result.Pipelines, runPipelineResult{
|
||||||
|
ID: pipeline.ID,
|
||||||
|
SourceBackend: pipeline.Source.Backend,
|
||||||
|
BundleCount: len(bundles),
|
||||||
|
Destinations: destinationIDs(pipeline.Destinations),
|
||||||
|
})
|
||||||
|
if options.Stdout != nil && !jsonOutput {
|
||||||
if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
|
if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
|
||||||
closeBackend(sourceBackend)
|
closeBackend(sourceBackend)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, sourceBundle := range bundles {
|
for _, destination := range pipeline.Destinations {
|
||||||
for _, destination := range pipeline.Destinations {
|
selections := selectDestinationBundles(destination, bundles)
|
||||||
destinationBackend, err := backends.openDestination(ctx, destination)
|
if isFixedPathDestination(destination) {
|
||||||
if err != nil {
|
summary.recordFixedPath()
|
||||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
if options.DryRun {
|
||||||
|
warning := fixedPathSelectionWarning(pipeline.ID, destination.ID, selections, len(bundles))
|
||||||
|
if jsonOutput {
|
||||||
|
warnings = append(warnings, warning)
|
||||||
|
} else if options.Stdout != nil {
|
||||||
|
if err := writeWarnings(options.Stdout, []OutputWarning{warning}); err != nil {
|
||||||
|
closeBackend(sourceBackend)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(selections) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
destinationBackend, err := backends.openDestination(ctx, destination)
|
||||||
|
if err != nil {
|
||||||
|
for _, selection := range selections {
|
||||||
|
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(selection.SourceBundle.RootRelativePath), err)
|
||||||
summary.recordFailure()
|
summary.recordFailure()
|
||||||
if options.Stdout != nil {
|
if jsonOutput {
|
||||||
writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, destination.Backend, err)
|
result.Actions = append(result.Actions, errorAction(pipeline.ID, destination.ID, destination.Backend, selection.SourceBundle.RootRelativePath, err))
|
||||||
}
|
} else if options.Stdout != nil {
|
||||||
continue
|
writeErrorLine(options.Stdout, selection.SourceBundle.RootRelativePath, destination.ID, destination.Backend, err)
|
||||||
}
|
|
||||||
closeDestination := true
|
|
||||||
deferCloseDestination := func() {
|
|
||||||
if closeDestination {
|
|
||||||
closeBackend(destinationBackend)
|
|
||||||
closeDestination = false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
closeDestination := true
|
||||||
|
deferCloseDestination := func() {
|
||||||
|
if closeDestination {
|
||||||
|
closeBackend(destinationBackend)
|
||||||
|
closeDestination = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, selection := range selections {
|
||||||
|
sourceBundle := selection.SourceBundle
|
||||||
req := publish.Request{
|
req := publish.Request{
|
||||||
PipelineID: pipeline.ID,
|
PipelineID: pipeline.ID,
|
||||||
DestinationID: destination.ID,
|
DestinationID: destination.ID,
|
||||||
SourceBundle: sourceBundle,
|
SourceBundle: sourceBundle,
|
||||||
SourceBackend: sourceBackend,
|
SourceBackend: sourceBackend,
|
||||||
DestinationBackend: destinationBackend,
|
DestinationBackend: destinationBackend,
|
||||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
DestinationBundlePath: selection.DestinationBundlePath,
|
||||||
|
PathMapping: destination.PathMap.Mode,
|
||||||
Publish: *destination.Publish,
|
Publish: *destination.Publish,
|
||||||
Transform: destination.Transform,
|
Transform: destination.Transform,
|
||||||
|
Links: destination.Links,
|
||||||
Transformers: transforms,
|
Transformers: transforms,
|
||||||
Transfer: destination.Transfer,
|
Transfer: destination.Transfer,
|
||||||
DistributorVersion: Version,
|
DistributorVersion: Version,
|
||||||
Force: options.Force,
|
Force: options.Force,
|
||||||
}
|
}
|
||||||
plan, err := publish.Build(ctx, req)
|
plan, err := publish.Build(ctx, req)
|
||||||
if err != nil && plan.DestinationID == "" {
|
if err != nil {
|
||||||
plan = publish.Plan{DestinationID: destination.ID, BundlePath: sourceBundle.RootRelativePath}
|
if plan.PipelineID == "" {
|
||||||
|
plan.PipelineID = pipeline.ID
|
||||||
|
}
|
||||||
|
if plan.DestinationID == "" {
|
||||||
|
plan.DestinationID = destination.ID
|
||||||
|
}
|
||||||
|
if plan.BundleID == "" {
|
||||||
|
plan.BundleID = sourceBundle.Manifest.ID
|
||||||
|
}
|
||||||
|
if plan.BundlePath == "" {
|
||||||
|
plan.BundlePath = sourceBundle.RootRelativePath
|
||||||
|
}
|
||||||
|
if plan.DestinationBundlePath == "" {
|
||||||
|
plan.DestinationBundlePath = selection.DestinationBundlePath
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if options.Stdout != nil {
|
if isFixedPathDestination(destination) {
|
||||||
|
plan.PathMapping = config.PathMappingFixed
|
||||||
|
if options.DryRun && isDestructiveFixedPathAction(plan.Action) {
|
||||||
|
warning := fixedPathReplacementWarning(plan)
|
||||||
|
if jsonOutput {
|
||||||
|
warnings = append(warnings, warning)
|
||||||
|
} else if options.Stdout != nil {
|
||||||
|
if err := writeWarnings(options.Stdout, []OutputWarning{warning}); err != nil {
|
||||||
|
deferCloseDestination()
|
||||||
|
closeBackend(sourceBackend)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if jsonOutput {
|
||||||
|
result.Actions = append(result.Actions, runActionFromPlan(destination.Backend, plan, err))
|
||||||
|
} else if options.Stdout != nil {
|
||||||
writePlanLine(options.Stdout, destination.Backend, plan, err)
|
writePlanLine(options.Stdout, destination.Backend, plan, err)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
deferCloseDestination()
|
|
||||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||||
summary.recordFailure()
|
summary.recordFailure()
|
||||||
continue
|
continue
|
||||||
@@ -137,26 +213,29 @@ func runConfigWithBackendFactory(ctx context.Context, cfg config.Config, options
|
|||||||
summary.recordPlan(plan.Action)
|
summary.recordPlan(plan.Action)
|
||||||
if !options.DryRun {
|
if !options.DryRun {
|
||||||
if err := publish.Execute(ctx, req, plan); err != nil {
|
if err := publish.Execute(ctx, req, plan); err != nil {
|
||||||
deferCloseDestination()
|
|
||||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||||
summary.recordFailure()
|
summary.recordFailure()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if shouldNotify(plan.Action) {
|
if shouldNotify(plan.Action) {
|
||||||
if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
|
if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
|
||||||
deferCloseDestination()
|
|
||||||
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
failures.add(pipeline.ID, destination.ID, destination.Backend, storage.DisplayPath(sourceBundle.RootRelativePath), err)
|
||||||
summary.recordFailure()
|
summary.recordFailure()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
deferCloseDestination()
|
|
||||||
}
|
}
|
||||||
|
deferCloseDestination()
|
||||||
}
|
}
|
||||||
closeBackend(sourceBackend)
|
closeBackend(sourceBackend)
|
||||||
}
|
}
|
||||||
if options.Stdout != nil {
|
result.Summary = summary.Result()
|
||||||
|
if jsonOutput {
|
||||||
|
if err := WriteJSONEnvelope(options.Stdout, "run", len(failures.items) == 0, warnings, result, failures.outputErrors()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if options.Stdout != nil {
|
||||||
if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil {
|
if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -178,175 +257,3 @@ func closeBackend(backend storage.Backend) {
|
|||||||
}
|
}
|
||||||
_ = closeable.Close()
|
_ = closeable.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error) {
|
|
||||||
if w == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if planErr != nil {
|
|
||||||
destinationID := plan.DestinationID
|
|
||||||
if destinationID == "" {
|
|
||||||
destinationID = "unknown"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, planErr.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, plan.Action, outputSummary(plan.Outputs), plan.Reason)
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
|
|
||||||
if w == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, backend, err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
func outputSummary(outputs []publish.Output) string {
|
|
||||||
if len(outputs) == 0 {
|
|
||||||
return "none"
|
|
||||||
}
|
|
||||||
paths := make([]string, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
paths = append(paths, output.DestinationPath)
|
|
||||||
}
|
|
||||||
return strings.Join(paths, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
func destinationSummary(destinations []config.Destination) string {
|
|
||||||
if len(destinations) == 0 {
|
|
||||||
return "none"
|
|
||||||
}
|
|
||||||
ids := make([]string, 0, len(destinations))
|
|
||||||
for _, destination := range destinations {
|
|
||||||
ids = append(ids, destination.ID)
|
|
||||||
}
|
|
||||||
return strings.Join(ids, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeSecretConflictWarnings(w io.Writer, conflicts []config.SecretConflict) error {
|
|
||||||
for _, conflict := range conflicts {
|
|
||||||
if _, err := fmt.Fprintf(w, "Warning: secret %s ignored because the real environment already has that variable\n", conflict.Name); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeSSHWarnings(w io.Writer, pipeline config.Pipeline) error {
|
|
||||||
if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
|
||||||
if _, err := fmt.Fprintf(w, "Warning: pipeline=%s source host_key_policy=off disables SSH host key checking\n", pipeline.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, destination := range pipeline.Destinations {
|
|
||||||
if destination.Backend == config.BackendSSH && destination.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
|
||||||
if _, err := fmt.Fprintf(w, "Warning: pipeline=%s destination=%s host_key_policy=off disables SSH host key checking\n", pipeline.ID, destination.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func shouldNotify(action publish.Action) bool {
|
|
||||||
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
|
||||||
}
|
|
||||||
|
|
||||||
func notifyEvent(plan publish.Plan) notify.Event {
|
|
||||||
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
|
||||||
for _, output := range plan.Outputs {
|
|
||||||
outputs = append(outputs, notify.Output{
|
|
||||||
Path: output.DestinationPath,
|
|
||||||
Kind: output.Kind,
|
|
||||||
SourcePath: output.SourcePath,
|
|
||||||
Transform: output.Transform,
|
|
||||||
SHA256: output.SHA256,
|
|
||||||
Size: output.Size,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return notify.Event{
|
|
||||||
PipelineID: plan.PipelineID,
|
|
||||||
DestinationID: plan.DestinationID,
|
|
||||||
BundleID: plan.BundleID,
|
|
||||||
BundlePath: plan.BundlePath,
|
|
||||||
Action: string(plan.Action),
|
|
||||||
Outputs: outputs,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type runSummary struct {
|
|
||||||
dryRun bool
|
|
||||||
planned int
|
|
||||||
publishNew int
|
|
||||||
replaceOlder int
|
|
||||||
forceReplace int
|
|
||||||
skipped int
|
|
||||||
failures int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *runSummary) recordPlan(action publish.Action) {
|
|
||||||
s.planned++
|
|
||||||
switch action {
|
|
||||||
case publish.ActionPublishNew:
|
|
||||||
s.publishNew++
|
|
||||||
case publish.ActionReplaceOlder:
|
|
||||||
s.replaceOlder++
|
|
||||||
case publish.ActionForceReplace:
|
|
||||||
s.forceReplace++
|
|
||||||
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
|
||||||
s.skipped++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *runSummary) recordFailure() {
|
|
||||||
s.failures++
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s runSummary) Line() string {
|
|
||||||
status := "ok"
|
|
||||||
if s.failures > 0 {
|
|
||||||
status = "failed"
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun)
|
|
||||||
}
|
|
||||||
|
|
||||||
type runFailure struct {
|
|
||||||
pipelineID string
|
|
||||||
destinationID string
|
|
||||||
backend string
|
|
||||||
bundlePath string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
type runFailures struct {
|
|
||||||
items []runFailure
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *runFailures) add(pipelineID, destinationID, backend, bundlePath string, err error) {
|
|
||||||
f.items = append(f.items, runFailure{
|
|
||||||
pipelineID: pipelineID,
|
|
||||||
destinationID: destinationID,
|
|
||||||
backend: backend,
|
|
||||||
bundlePath: bundlePath,
|
|
||||||
err: err,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f runFailures) Error() string {
|
|
||||||
if len(f.items) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
parts := make([]string, 0, len(f.items))
|
|
||||||
for _, item := range f.items {
|
|
||||||
parts = append(parts, fmt.Sprintf("pipeline %s destination %s backend %s bundle %s: %v", item.pipelineID, item.destinationID, item.backend, item.bundlePath, item.err))
|
|
||||||
}
|
|
||||||
return "run failed: " + strings.Join(parts, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f runFailures) Unwrap() error {
|
|
||||||
errs := make([]error, 0, len(f.items))
|
|
||||||
for _, item := range f.items {
|
|
||||||
errs = append(errs, item.err)
|
|
||||||
}
|
|
||||||
return errors.Join(errs...)
|
|
||||||
}
|
|
||||||
|
|||||||
70
internal/app/run_failures.go
Normal file
70
internal/app/run_failures.go
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type runFailure struct {
|
||||||
|
pipelineID string
|
||||||
|
destinationID string
|
||||||
|
backend string
|
||||||
|
bundlePath string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type runFailures struct {
|
||||||
|
items []runFailure
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *runFailures) add(pipelineID, destinationID, backend, bundlePath string, err error) {
|
||||||
|
f.items = append(f.items, runFailure{
|
||||||
|
pipelineID: pipelineID,
|
||||||
|
destinationID: destinationID,
|
||||||
|
backend: backend,
|
||||||
|
bundlePath: bundlePath,
|
||||||
|
err: err,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f runFailures) Error() string {
|
||||||
|
if len(f.items) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
parts := make([]string, 0, len(f.items))
|
||||||
|
for _, item := range f.items {
|
||||||
|
parts = append(parts, fmt.Sprintf("pipeline %s destination %s backend %s bundle %s: %v", item.pipelineID, item.destinationID, item.backend, item.bundlePath, item.err))
|
||||||
|
}
|
||||||
|
return "run failed: " + strings.Join(parts, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f runFailures) outputErrors() []OutputError {
|
||||||
|
if len(f.items) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
errors := make([]OutputError, 0, len(f.items))
|
||||||
|
for _, item := range f.items {
|
||||||
|
errors = append(errors, OutputError{
|
||||||
|
PipelineID: item.pipelineID,
|
||||||
|
DestinationID: item.destinationID,
|
||||||
|
Backend: item.backend,
|
||||||
|
BundlePath: item.bundlePath,
|
||||||
|
Message: item.err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsPartialResultError(err error) bool {
|
||||||
|
var failures runFailures
|
||||||
|
return errors.As(err, &failures)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f runFailures) Unwrap() error {
|
||||||
|
errs := make([]error, 0, len(f.items))
|
||||||
|
for _, item := range f.items {
|
||||||
|
errs = append(errs, item.err)
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
33
internal/app/run_notify.go
Normal file
33
internal/app/run_notify.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
)
|
||||||
|
|
||||||
|
func shouldNotify(action publish.Action) bool {
|
||||||
|
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
||||||
|
}
|
||||||
|
|
||||||
|
func notifyEvent(plan publish.Plan) notify.Event {
|
||||||
|
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
||||||
|
for _, output := range plan.Outputs {
|
||||||
|
stateOutput := output.StateOutputFile()
|
||||||
|
outputs = append(outputs, notify.Output{
|
||||||
|
Path: stateOutput.Path,
|
||||||
|
Kind: stateOutput.Kind,
|
||||||
|
SourcePath: stateOutput.SourcePath,
|
||||||
|
Transform: stateOutput.Transform,
|
||||||
|
SHA256: stateOutput.SHA256,
|
||||||
|
Size: stateOutput.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return notify.Event{
|
||||||
|
PipelineID: plan.PipelineID,
|
||||||
|
DestinationID: plan.DestinationID,
|
||||||
|
BundleID: plan.BundleID,
|
||||||
|
BundlePath: plan.BundlePath,
|
||||||
|
Action: string(plan.Action),
|
||||||
|
Outputs: outputs,
|
||||||
|
}
|
||||||
|
}
|
||||||
154
internal/app/run_output.go
Normal file
154
internal/app/run_output.go
Normal file
@@ -0,0 +1,154 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writePlanLine(w io.Writer, backend string, plan publish.Plan, planErr error) {
|
||||||
|
if w == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if planErr != nil {
|
||||||
|
destinationID := plan.DestinationID
|
||||||
|
if destinationID == "" {
|
||||||
|
destinationID = "unknown"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", storage.DisplayPath(plan.BundlePath), destinationID, backend, pathMappingSummary(plan), planErr.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", storage.DisplayPath(plan.BundlePath), plan.DestinationID, backend, pathMappingSummary(plan), plan.Action, outputSummary(plan.Outputs), plan.Reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathMappingSummary(plan publish.Plan) string {
|
||||||
|
if plan.PathMapping != config.PathMappingFixed {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(" path_mapping=fixed target=%s", storage.DisplayPath(plan.DestinationBundlePath))
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeErrorLine(w io.Writer, bundlePath, destinationID, backend string, err error) {
|
||||||
|
if w == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s action=error reason=%q\n", storage.DisplayPath(bundlePath), destinationID, backend, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
func outputSummary(outputs []publish.Output) string {
|
||||||
|
if len(outputs) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
paths := make([]string, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
paths = append(paths, output.DestinationPath)
|
||||||
|
}
|
||||||
|
return strings.Join(paths, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
type runResult struct {
|
||||||
|
DryRun bool `json:"dry_run"`
|
||||||
|
Pipelines []runPipelineResult `json:"pipelines"`
|
||||||
|
Actions []runActionResult `json:"actions"`
|
||||||
|
Summary runSummaryResult `json:"summary"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runPipelineResult struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
SourceBackend string `json:"source_backend"`
|
||||||
|
BundleCount int `json:"bundle_count"`
|
||||||
|
Destinations []string `json:"destinations"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runActionResult struct {
|
||||||
|
PipelineID string `json:"pipeline_id,omitempty"`
|
||||||
|
DestinationID string `json:"destination_id"`
|
||||||
|
Backend string `json:"backend"`
|
||||||
|
BundleID string `json:"bundle_id,omitempty"`
|
||||||
|
BundlePath string `json:"bundle_path"`
|
||||||
|
DestinationPath string `json:"destination_path"`
|
||||||
|
PathMapping string `json:"path_mapping,omitempty"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
PrimaryURL string `json:"primary_url,omitempty"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
Outputs []runOutputResult `json:"outputs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type runOutputResult struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
SourcePath string `json:"source_path,omitempty"`
|
||||||
|
Transform string `json:"transform,omitempty"`
|
||||||
|
URL string `json:"url,omitempty"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func runActionFromPlan(backend string, plan publish.Plan, planErr error) runActionResult {
|
||||||
|
if planErr != nil {
|
||||||
|
destinationID := plan.DestinationID
|
||||||
|
if destinationID == "" {
|
||||||
|
destinationID = "unknown"
|
||||||
|
}
|
||||||
|
return runActionResult{
|
||||||
|
PipelineID: plan.PipelineID,
|
||||||
|
DestinationID: destinationID,
|
||||||
|
Backend: backend,
|
||||||
|
BundleID: plan.BundleID,
|
||||||
|
BundlePath: storage.DisplayPath(plan.BundlePath),
|
||||||
|
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
||||||
|
PathMapping: plan.PathMapping,
|
||||||
|
Action: "error",
|
||||||
|
PrimaryURL: plan.PrimaryURL,
|
||||||
|
Reason: planErr.Error(),
|
||||||
|
Outputs: []runOutputResult{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return runActionResult{
|
||||||
|
PipelineID: plan.PipelineID,
|
||||||
|
DestinationID: plan.DestinationID,
|
||||||
|
Backend: backend,
|
||||||
|
BundleID: plan.BundleID,
|
||||||
|
BundlePath: storage.DisplayPath(plan.BundlePath),
|
||||||
|
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
||||||
|
PathMapping: plan.PathMapping,
|
||||||
|
Action: string(plan.Action),
|
||||||
|
PrimaryURL: plan.PrimaryURL,
|
||||||
|
Reason: plan.Reason,
|
||||||
|
Outputs: runOutputsFromPlan(plan.Outputs),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func errorAction(pipelineID, destinationID, backend, bundlePath string, err error) runActionResult {
|
||||||
|
return runActionResult{
|
||||||
|
PipelineID: pipelineID,
|
||||||
|
DestinationID: destinationID,
|
||||||
|
Backend: backend,
|
||||||
|
BundlePath: storage.DisplayPath(bundlePath),
|
||||||
|
DestinationPath: storage.DisplayPath(bundlePath),
|
||||||
|
Action: "error",
|
||||||
|
Reason: err.Error(),
|
||||||
|
Outputs: []runOutputResult{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runOutputsFromPlan(outputs []publish.Output) []runOutputResult {
|
||||||
|
results := make([]runOutputResult, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
stateOutput := output.StateOutputFile()
|
||||||
|
results = append(results, runOutputResult{
|
||||||
|
Path: stateOutput.Path,
|
||||||
|
Kind: stateOutput.Kind,
|
||||||
|
SourcePath: stateOutput.SourcePath,
|
||||||
|
Transform: stateOutput.Transform,
|
||||||
|
URL: stateOutput.URL,
|
||||||
|
SHA256: stateOutput.SHA256,
|
||||||
|
Size: stateOutput.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
91
internal/app/run_selection.go
Normal file
91
internal/app/run_selection.go
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type destinationBundleSelection struct {
|
||||||
|
SourceBundle bundle.Bundle
|
||||||
|
DestinationBundlePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectDestinationBundles(destination config.Destination, bundles []bundle.Bundle) []destinationBundleSelection {
|
||||||
|
if !isFixedPathDestination(destination) {
|
||||||
|
selections := make([]destinationBundleSelection, 0, len(bundles))
|
||||||
|
for _, sourceBundle := range bundles {
|
||||||
|
selections = append(selections, destinationBundleSelection{
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return selections
|
||||||
|
}
|
||||||
|
if len(bundles) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sourceBundle := newestBundle(bundles)
|
||||||
|
return []destinationBundleSelection{{
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
DestinationBundlePath: "",
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newestBundle(bundles []bundle.Bundle) bundle.Bundle {
|
||||||
|
if len(bundles) == 0 {
|
||||||
|
return bundle.Bundle{}
|
||||||
|
}
|
||||||
|
sorted := append([]bundle.Bundle(nil), bundles...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool {
|
||||||
|
if sorted[i].Manifest.Created.Equal(sorted[j].Manifest.Created) {
|
||||||
|
return sorted[i].RootRelativePath < sorted[j].RootRelativePath
|
||||||
|
}
|
||||||
|
return sorted[i].Manifest.Created.After(sorted[j].Manifest.Created)
|
||||||
|
})
|
||||||
|
return sorted[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func isFixedPathDestination(destination config.Destination) bool {
|
||||||
|
return destination.PathMap.Mode == config.PathMappingFixed
|
||||||
|
}
|
||||||
|
|
||||||
|
func fixedPathSelectionWarning(pipelineID, destinationID string, selections []destinationBundleSelection, candidateCount int) OutputWarning {
|
||||||
|
selected := "none"
|
||||||
|
if len(selections) > 0 {
|
||||||
|
selected = storage.DisplayPath(selections[0].SourceBundle.RootRelativePath)
|
||||||
|
}
|
||||||
|
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isDestructiveFixedPathAction(action publish.Action) bool {
|
||||||
|
return action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
||||||
|
}
|
||||||
|
|
||||||
|
func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
|
||||||
|
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||||
|
}
|
||||||
|
|
||||||
|
func destinationIDs(destinations []config.Destination) []string {
|
||||||
|
ids := make([]string, 0, len(destinations))
|
||||||
|
for _, destination := range destinations {
|
||||||
|
ids = append(ids, destination.ID)
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func destinationSummary(destinations []config.Destination) string {
|
||||||
|
if len(destinations) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
ids := make([]string, 0, len(destinations))
|
||||||
|
for _, destination := range destinations {
|
||||||
|
ids = append(ids, destination.ID)
|
||||||
|
}
|
||||||
|
return strings.Join(ids, ",")
|
||||||
|
}
|
||||||
78
internal/app/run_summary.go
Normal file
78
internal/app/run_summary.go
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
)
|
||||||
|
|
||||||
|
type runSummary struct {
|
||||||
|
dryRun bool
|
||||||
|
planned int
|
||||||
|
publishNew int
|
||||||
|
replaceOlder int
|
||||||
|
forceReplace int
|
||||||
|
skipped int
|
||||||
|
failures int
|
||||||
|
fixedPath int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *runSummary) recordPlan(action publish.Action) {
|
||||||
|
s.planned++
|
||||||
|
switch action {
|
||||||
|
case publish.ActionPublishNew:
|
||||||
|
s.publishNew++
|
||||||
|
case publish.ActionReplaceOlder:
|
||||||
|
s.replaceOlder++
|
||||||
|
case publish.ActionForceReplace:
|
||||||
|
s.forceReplace++
|
||||||
|
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
||||||
|
s.skipped++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *runSummary) recordFailure() {
|
||||||
|
s.failures++
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *runSummary) recordFixedPath() {
|
||||||
|
s.fixedPath++
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s runSummary) Line() string {
|
||||||
|
status := "ok"
|
||||||
|
if s.failures > 0 {
|
||||||
|
status = "failed"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", status, s.planned, s.publishNew, s.replaceOlder, s.forceReplace, s.skipped, s.failures, s.dryRun, s.fixedPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
type runSummaryResult struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Planned int `json:"planned"`
|
||||||
|
PublishNew int `json:"publish_new"`
|
||||||
|
ReplaceOlder int `json:"replace_older"`
|
||||||
|
ForceReplace int `json:"force_replace"`
|
||||||
|
Skipped int `json:"skipped"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
DryRun bool `json:"dry_run"`
|
||||||
|
FixedPath int `json:"fixed_path"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s runSummary) Result() runSummaryResult {
|
||||||
|
status := "ok"
|
||||||
|
if s.failures > 0 {
|
||||||
|
status = "failed"
|
||||||
|
}
|
||||||
|
return runSummaryResult{
|
||||||
|
Status: status,
|
||||||
|
Planned: s.planned,
|
||||||
|
PublishNew: s.publishNew,
|
||||||
|
ReplaceOlder: s.replaceOlder,
|
||||||
|
ForceReplace: s.forceReplace,
|
||||||
|
Skipped: s.skipped,
|
||||||
|
Failed: s.failures,
|
||||||
|
DryRun: s.dryRun,
|
||||||
|
FixedPath: s.fixedPath,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -159,9 +159,9 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteSSHWarningsReportsInsecureHostKeyPolicy(t *testing.T) {
|
func TestSSHWarningsReportInsecureHostKeyPolicy(t *testing.T) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
err := writeSSHWarnings(&stdout, config.Pipeline{
|
err := writeWarnings(&stdout, sshWarnings(config.Pipeline{
|
||||||
ID: "reports",
|
ID: "reports",
|
||||||
Source: config.Backend{
|
Source: config.Backend{
|
||||||
Backend: config.BackendSSH,
|
Backend: config.BackendSSH,
|
||||||
@@ -172,9 +172,9 @@ func TestWriteSSHWarningsReportsInsecureHostKeyPolicy(t *testing.T) {
|
|||||||
Backend: config.BackendSSH,
|
Backend: config.BackendSSH,
|
||||||
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff},
|
SSH: config.SSH{HostKeyPolicy: config.HostKeyPolicyOff},
|
||||||
}},
|
}},
|
||||||
})
|
}))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("writeSSHWarnings() error = %v", err)
|
t.Fatalf("writeWarnings() error = %v", err)
|
||||||
}
|
}
|
||||||
output := stdout.String()
|
output := stdout.String()
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
@@ -200,8 +200,8 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
assertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||||
if _, err := os.Stat(filepath.Join(destinationRoot, "manifest.json")); !os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(destinationRoot, "manifest.json")); !os.IsNotExist(err) {
|
||||||
t.Fatalf("destination manifest stat error = %v, want not exist", err)
|
t.Fatalf("destination manifest stat error = %v, want not exist", err)
|
||||||
}
|
}
|
||||||
@@ -215,6 +215,380 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
|||||||
if got, want := len(destinationState.Outputs), 2; got != want {
|
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||||
t.Fatalf("state output count = %d, want %d", got, want)
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
}
|
}
|
||||||
|
if destinationState.Links != nil || destinationState.Outputs[0].URL != "" {
|
||||||
|
t.Fatalf("state links = %#v output URL=%q, want absent", destinationState.Links, destinationState.Outputs[0].URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunExplicitPreserveRelativePathMappingMatchesDefault(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "daily/report", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "daily", "report", "report.md"), "# Report\nSunny.\n")
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("root report.md stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordsLinksForNestedBundlePath(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "daily/brentwood", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, "https://reports.example.com/archive", config.LinkPrimaryAuto, true, false, ""),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName))
|
||||||
|
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||||
|
t.Fatalf("state links = %#v, want source primary URL", destinationState.Links)
|
||||||
|
}
|
||||||
|
outputs := outputsByPath(destinationState.Outputs)
|
||||||
|
if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||||
|
t.Fatalf("report URL = %q", outputs["report.md"].URL)
|
||||||
|
}
|
||||||
|
if outputs["summary.txt"].URL != "https://reports.example.com/archive/daily/brentwood/summary.txt" {
|
||||||
|
t.Fatalf("summary URL = %q", outputs["summary.txt"].URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordsLinksForFixedIndexDestination(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "older", testBundleOptions{ID: "reports.older", Created: testutil.DefaultCreated})
|
||||||
|
writeSourceBundle(t, sourceRoot, "newer", testBundleOptions{ID: "reports.newer", Created: testutil.DefaultCreated.Add(time.Hour)})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingFixed, "https://reports.example.com/latest", config.LinkPrimaryAuto, false, true, config.TransformModeIndex),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
|
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/latest/" {
|
||||||
|
t.Fatalf("state links = %#v, want fixed index primary URL", destinationState.Links)
|
||||||
|
}
|
||||||
|
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||||
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
if destinationState.Outputs[0].Path != "index.html" || destinationState.Outputs[0].URL != "https://reports.example.com/latest/" {
|
||||||
|
t.Fatalf("state output = %#v, want index URL", destinationState.Outputs[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathPublishesNewestBundleAtDestinationRoot(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
|
||||||
|
ID: "reports.old",
|
||||||
|
Created: testutil.DefaultCreated,
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nOld.\n"},
|
||||||
|
{Path: "summary.txt", Data: "Old summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
|
||||||
|
ID: "reports.new",
|
||||||
|
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||||
|
{Path: "summary.txt", Data: "New summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, "new", "report.md")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("nested new report stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
|
if destinationState.Source.Manifest.ID != "reports.new" {
|
||||||
|
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathTieBreaksByBundlePath(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "b", testBundleOptions{
|
||||||
|
ID: "reports.b",
|
||||||
|
Created: testutil.DefaultCreated,
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nB.\n"},
|
||||||
|
{Path: "summary.txt", Data: "B summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
writeSourceBundle(t, sourceRoot, "a", testBundleOptions{
|
||||||
|
ID: "reports.a",
|
||||||
|
Created: testutil.DefaultCreated,
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nA.\n"},
|
||||||
|
{Path: "summary.txt", Data: "A summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
|
if destinationState.Source.Manifest.ID != "reports.a" {
|
||||||
|
t.Fatalf("state source id = %q, want reports.a", destinationState.Source.Manifest.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathDryRunReportsSelection(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{ID: "reports.old", Created: testutil.DefaultCreated})
|
||||||
|
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{ID: "reports.new", Created: testutil.DefaultCreated.Add(time.Hour)})
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
|
||||||
|
DryRun: true,
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
output := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"Warning: pipeline=reports destination=archive path_mapping=fixed candidates=2 selected_bundle=new destination_bundle=.",
|
||||||
|
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=publish_new",
|
||||||
|
"fixed_path=1",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(output, want) {
|
||||||
|
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(output, "bundle=old destination=archive") {
|
||||||
|
t.Fatalf("stdout = %q, older fixed candidate was planned", output)
|
||||||
|
}
|
||||||
|
if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
|
||||||
|
t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
|
||||||
|
ID: "reports.old",
|
||||||
|
Created: testutil.DefaultCreated,
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nOld.\n"},
|
||||||
|
{Path: "summary.txt", Data: "Old summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
|
||||||
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
|
t.Fatalf("first Run() error = %v", err)
|
||||||
|
}
|
||||||
|
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
|
||||||
|
ID: "reports.new",
|
||||||
|
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||||
|
{Path: "summary.txt", Data: "New summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: configPath,
|
||||||
|
DryRun: true,
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
output := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"Warning: pipeline=reports destination=archive path_mapping=fixed action=replace_older replaces destination root for selected_bundle=new",
|
||||||
|
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_older",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(output, want) {
|
||||||
|
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
|
||||||
|
ID: "reports.old",
|
||||||
|
Created: testutil.DefaultCreated,
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nOld.\n"},
|
||||||
|
{Path: "summary.txt", Data: "Old summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
|
||||||
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
|
t.Fatalf("first Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
|
||||||
|
|
||||||
|
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
|
||||||
|
ID: "reports.new",
|
||||||
|
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||||
|
{Path: "summary.txt", Data: "New summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
|
t.Fatalf("second Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
|
if destinationState.Source.Manifest.ID != "reports.new" {
|
||||||
|
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathSkipsWhenDestinationStateIsNewer(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
newer := testutil.ValidManifest(testutil.BundleOptions{
|
||||||
|
ID: "reports.newer",
|
||||||
|
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||||
|
})
|
||||||
|
writeDestinationState(t, destinationRoot, "", newer)
|
||||||
|
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nExisting.\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write existing report: %v", err)
|
||||||
|
}
|
||||||
|
writeSourceBundle(t, sourceRoot, "older", testBundleOptions{
|
||||||
|
ID: "reports.older",
|
||||||
|
Created: testutil.DefaultCreated,
|
||||||
|
})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
|
||||||
|
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
|
||||||
|
}
|
||||||
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nExisting.\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{})
|
||||||
|
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write unmanaged file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
|
||||||
|
t.Fatalf("Run() error = %v, want unmanaged failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
parent := t.TempDir()
|
||||||
|
destinationRoot := filepath.Join(parent, "latest")
|
||||||
|
if err := os.MkdirAll(destinationRoot, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir destination: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(parent, "keep.txt"), []byte("keep"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write sibling: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write unmanaged: %v", err)
|
||||||
|
}
|
||||||
|
writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
|
||||||
|
Force: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
|
testutil.AssertFile(t, filepath.Join(parent, "keep.txt"), "keep")
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unmanaged stat error = %v, want removed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
|
||||||
|
localSourceRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, localSourceRoot, "old", testBundleOptions{
|
||||||
|
ID: "reports.old",
|
||||||
|
Created: testutil.DefaultCreated,
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nOld.\n"},
|
||||||
|
{Path: "summary.txt", Data: "Old summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
writeSourceBundle(t, localSourceRoot, "new", testBundleOptions{
|
||||||
|
ID: "reports.new",
|
||||||
|
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||||
|
{Path: "summary.txt", Data: "New summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
s3Destination := fake.New()
|
||||||
|
sshDestination := fake.New()
|
||||||
|
cfg := config.Config{Pipelines: []config.Pipeline{{
|
||||||
|
ID: "reports",
|
||||||
|
Source: config.Backend{Backend: config.BackendLocal, Path: localSourceRoot},
|
||||||
|
Destinations: []config.Destination{
|
||||||
|
{
|
||||||
|
ID: "object-latest",
|
||||||
|
Backend: config.BackendS3,
|
||||||
|
Endpoint: "http://s3.test",
|
||||||
|
Bucket: "destination-bucket",
|
||||||
|
PathMap: config.PathMapping{Mode: config.PathMappingFixed},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "ssh-latest",
|
||||||
|
Backend: config.BackendSSH,
|
||||||
|
Host: "ssh.test",
|
||||||
|
Path: "/latest",
|
||||||
|
PathMap: config.PathMapping{Mode: config.PathMappingFixed},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}}
|
||||||
|
config.ApplyDefaults(&cfg)
|
||||||
|
provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
|
||||||
|
"s3:destination-bucket": s3Destination,
|
||||||
|
"ssh:/latest": sshDestination,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{}, provider); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nNew.\n")
|
||||||
|
testutil.AssertFakeFile(t, s3Destination, "summary.txt", "New summary\n")
|
||||||
|
testutil.AssertFakeMissing(t, s3Destination, "new/report.md")
|
||||||
|
testutil.AssertFakeFile(t, sshDestination, "report.md", "# Report\nNew.\n")
|
||||||
|
testutil.AssertFakeFile(t, sshDestination, "summary.txt", "New summary\n")
|
||||||
|
testutil.AssertFakeMissing(t, sshDestination, "new/report.md")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunNotifiesAfterPublication(t *testing.T) {
|
func TestRunNotifiesAfterPublication(t *testing.T) {
|
||||||
@@ -248,6 +622,32 @@ func TestRunNotifiesAfterPublication(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunNotifiesGeneratedOutputMetadata(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
notifier := &recordingNotifier{}
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, false, true),
|
||||||
|
Notifier: notifier,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := len(notifier.events), 1; got != want {
|
||||||
|
t.Fatalf("notification count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
outputs := notifier.events[0].Outputs
|
||||||
|
if got, want := len(outputs), 1; got != want {
|
||||||
|
t.Fatalf("notification output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
output := outputs[0]
|
||||||
|
if output.Path != "report.html" || output.Kind != state.OutputKindGenerated || output.SourcePath != "report.md" || output.Transform != "markdown_to_html" || output.SHA256 == "" || output.Size <= 0 {
|
||||||
|
t.Fatalf("notification output = %#v", output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunNotifiesAfterReplacement(t *testing.T) {
|
func TestRunNotifiesAfterReplacement(t *testing.T) {
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
@@ -275,6 +675,49 @@ func TestRunNotifiesAfterReplacement(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunJSONIncludesGeneratedOutputMetadata(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: testutil.WriteLocalConfigWithLinks(t, sourceRoot, destinationRoot, config.PathMappingFixed, "https://reports.example.com/latest", config.LinkPrimaryAuto, false, true, config.TransformModeIndex),
|
||||||
|
DryRun: true,
|
||||||
|
Stdout: &stdout,
|
||||||
|
OutputFormat: OutputFormatJSON,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
result := decodeAppResult(t, stdout.String())
|
||||||
|
actions, ok := result["actions"].([]any)
|
||||||
|
if !ok || len(actions) != 1 {
|
||||||
|
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||||
|
}
|
||||||
|
action, ok := actions[0].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("action = %#v, want object", actions[0])
|
||||||
|
}
|
||||||
|
if action["primary_url"] != "https://reports.example.com/latest/" {
|
||||||
|
t.Fatalf("action primary_url = %#v", action["primary_url"])
|
||||||
|
}
|
||||||
|
outputs, ok := action["outputs"].([]any)
|
||||||
|
if !ok || len(outputs) != 1 {
|
||||||
|
t.Fatalf("outputs = %#v, want one output", action["outputs"])
|
||||||
|
}
|
||||||
|
output, ok := outputs[0].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("output = %#v, want object", outputs[0])
|
||||||
|
}
|
||||||
|
if output["path"] != "index.html" || output["kind"] != state.OutputKindGenerated || output["source_path"] != "report.md" || output["transform"] != "markdown_to_html" || output["url"] != "https://reports.example.com/latest/" {
|
||||||
|
t.Fatalf("output = %#v, want generated index metadata", output)
|
||||||
|
}
|
||||||
|
if output["sha256"] == "" || output["size"] == nil {
|
||||||
|
t.Fatalf("output = %#v, want digest and size", output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
|
func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
@@ -346,7 +789,7 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
|
|||||||
t.Fatalf("stdout = %q, want substring %q", output, want)
|
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
assertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunPublishesHTMLOnly(t *testing.T) {
|
func TestRunPublishesHTMLOnly(t *testing.T) {
|
||||||
@@ -354,11 +797,11 @@ func TestRunPublishesHTMLOnly(t *testing.T) {
|
|||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, false, true)})
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, false, true)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
assertFileContains(t, filepath.Join(destinationRoot, "report.html"), "<h1>Report</h1>")
|
testutil.AssertFileContains(t, filepath.Join(destinationRoot, "report.html"), "<h1>Report</h1>")
|
||||||
if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) {
|
||||||
t.Fatalf("report.md stat error = %v, want not exist", err)
|
t.Fatalf("report.md stat error = %v, want not exist", err)
|
||||||
}
|
}
|
||||||
@@ -372,18 +815,71 @@ func TestRunPublishesHTMLOnly(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunPublishesHTMLIndexWithExplicitInput(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||||
|
ExtraFiles: []testFile{{Path: "notes.md", Data: "# Notes\nHidden.\n"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "report.md")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFileContains(t, filepath.Join(destinationRoot, "index.html"), "<h1>Report</h1>")
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, "report.html")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("report.html stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
|
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||||
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
output := destinationState.Outputs[0]
|
||||||
|
if output.Kind != state.OutputKindGenerated || output.Transform != "markdown_to_html" || output.Path != "index.html" || output.SourcePath != "report.md" {
|
||||||
|
t.Fatalf("generated output metadata = %#v", output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPublishesHTMLIndexWithSingleMarkdownFallback(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFileContains(t, filepath.Join(destinationRoot, "index.html"), "<h1>Report</h1>")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFailsIndexModeWithAmbiguousMarkdownInput(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||||
|
ExtraFiles: []testFile{{Path: "notes.md", Data: "# Notes\n"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "")})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "multiple markdown source files") {
|
||||||
|
t.Fatalf("Run() error = %v, want ambiguous input error", err)
|
||||||
|
}
|
||||||
|
if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
|
||||||
|
t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunPublishesSourceAndHTML(t *testing.T) {
|
func TestRunPublishesSourceAndHTML(t *testing.T) {
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)})
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, true, true)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
assertFileContains(t, filepath.Join(destinationRoot, "report.html"), "<p>Sunny.</p>")
|
testutil.AssertFileContains(t, filepath.Join(destinationRoot, "report.html"), "<p>Sunny.</p>")
|
||||||
assertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
if got, want := len(destinationState.Outputs), 3; got != want {
|
if got, want := len(destinationState.Outputs), 3; got != want {
|
||||||
t.Fatalf("state output count = %d, want %d", got, want)
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
@@ -400,7 +896,7 @@ func TestRunDoesNotMutateSourceBundle(t *testing.T) {
|
|||||||
t.Fatalf("read source before: %v", err)
|
t.Fatalf("read source before: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)})
|
err = Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, true, true)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -418,7 +914,23 @@ func TestRunFailsOnOutputPathCollision(t *testing.T) {
|
|||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{ExtraFiles: []testFile{{Path: "report.html", Data: "<p>source html</p>\n"}}})
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{ExtraFiles: []testFile{{Path: "report.html", Data: "<p>source html</p>\n"}}})
|
||||||
|
|
||||||
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)})
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, true, true)})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "destination output path collision") {
|
||||||
|
t.Fatalf("Run() error = %v, want collision", err)
|
||||||
|
}
|
||||||
|
if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
|
||||||
|
t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFailsOnIndexOutputPathCollision(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||||
|
ExtraFiles: []testFile{{Path: "index.html", Data: "<p>source index</p>\n"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, true, true, config.TransformModeIndex, "report.md")})
|
||||||
if err == nil || !strings.Contains(err.Error(), "destination output path collision") {
|
if err == nil || !strings.Contains(err.Error(), "destination output path collision") {
|
||||||
t.Fatalf("Run() error = %v, want collision", err)
|
t.Fatalf("Run() error = %v, want collision", err)
|
||||||
}
|
}
|
||||||
@@ -434,7 +946,7 @@ func TestRunDryRunReportsGeneratedOutputs(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
err := Run(context.Background(), RunOptions{
|
err := Run(context.Background(), RunOptions{
|
||||||
ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, false, true),
|
ConfigPath: testutil.WriteLocalConfigWithPublishPolicy(t, sourceRoot, destinationRoot, false, true),
|
||||||
DryRun: true,
|
DryRun: true,
|
||||||
Stdout: &stdout,
|
Stdout: &stdout,
|
||||||
})
|
})
|
||||||
@@ -446,6 +958,73 @@ func TestRunDryRunReportsGeneratedOutputs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunDryRunReportsIndexOutputWithoutWriting(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, ""),
|
||||||
|
DryRun: true,
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "outputs=index.html") {
|
||||||
|
t.Fatalf("stdout = %q, want index output path", stdout.String())
|
||||||
|
}
|
||||||
|
if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
|
||||||
|
t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSourceOnlyDoesNotWriteIndexOutput(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, true, false, config.TransformModeIndex, "")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, "index.html")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("index.html stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunReplacesHTMLIndexOutput(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||||
|
Created: testutil.DefaultCreated,
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nOld.\n"},
|
||||||
|
{Path: "summary.txt", Data: "Summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
configPath := testutil.WriteLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "")
|
||||||
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
|
t.Fatalf("first Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFileContains(t, filepath.Join(destinationRoot, "index.html"), "<p>Old.</p>")
|
||||||
|
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||||
|
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||||
|
{Path: "summary.txt", Data: "Summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
|
t.Fatalf("second Run() error = %v", err)
|
||||||
|
}
|
||||||
|
testutil.AssertFileContains(t, filepath.Join(destinationRoot, "index.html"), "<p>New.</p>")
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
@@ -484,7 +1063,7 @@ func TestRunReplacesOlderDestination(t *testing.T) {
|
|||||||
if !strings.Contains(stdout.String(), "action=replace_older") {
|
if !strings.Contains(stdout.String(), "action=replace_older") {
|
||||||
t.Fatalf("stdout = %q, want replace_older", stdout.String())
|
t.Fatalf("stdout = %q, want replace_older", stdout.String())
|
||||||
}
|
}
|
||||||
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSkipsNewerDestination(t *testing.T) {
|
func TestRunSkipsNewerDestination(t *testing.T) {
|
||||||
@@ -506,7 +1085,7 @@ func TestRunSkipsNewerDestination(t *testing.T) {
|
|||||||
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
|
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
|
||||||
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
|
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
|
||||||
}
|
}
|
||||||
assertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFailsOnConflict(t *testing.T) {
|
func TestRunFailsOnConflict(t *testing.T) {
|
||||||
@@ -559,7 +1138,7 @@ func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
|
|||||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
||||||
t.Fatalf("unmanaged file stat error = %v, want not exist", err)
|
t.Fatalf("unmanaged file stat error = %v, want not exist", err)
|
||||||
}
|
}
|
||||||
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFansOutToLocalDestinations(t *testing.T) {
|
func TestRunFansOutToLocalDestinations(t *testing.T) {
|
||||||
@@ -572,8 +1151,8 @@ func TestRunFansOutToLocalDestinations(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
assertFile(t, filepath.Join(firstDestination, "daily", "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(firstDestination, "daily", "report.md"), "# Report\nSunny.\n")
|
||||||
assertFile(t, filepath.Join(secondDestination, "daily", "summary.txt"), "Summary\n")
|
testutil.AssertFile(t, filepath.Join(secondDestination, "daily", "summary.txt"), "Summary\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFansOutWithDifferentPublishPolicies(t *testing.T) {
|
func TestRunFansOutWithDifferentPublishPolicies(t *testing.T) {
|
||||||
@@ -582,16 +1161,16 @@ func TestRunFansOutWithDifferentPublishPolicies(t *testing.T) {
|
|||||||
htmlDestination := t.TempDir()
|
htmlDestination := t.TempDir()
|
||||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
err := Run(context.Background(), RunOptions{ConfigPath: writeMixedPolicyFanoutConfig(t, sourceRoot, archiveDestination, htmlDestination)})
|
err := Run(context.Background(), RunOptions{ConfigPath: testutil.WriteMixedPolicyFanoutLocalConfig(t, sourceRoot, archiveDestination, htmlDestination)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
assertFile(t, filepath.Join(archiveDestination, "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(archiveDestination, "report.md"), "# Report\nSunny.\n")
|
||||||
assertFile(t, filepath.Join(archiveDestination, "summary.txt"), "Summary\n")
|
testutil.AssertFile(t, filepath.Join(archiveDestination, "summary.txt"), "Summary\n")
|
||||||
if _, err := os.Stat(filepath.Join(archiveDestination, "report.html")); !os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(archiveDestination, "report.html")); !os.IsNotExist(err) {
|
||||||
t.Fatalf("archive report.html stat error = %v, want not exist", err)
|
t.Fatalf("archive report.html stat error = %v, want not exist", err)
|
||||||
}
|
}
|
||||||
assertFileContains(t, filepath.Join(htmlDestination, "report.html"), "<h1>Report</h1>")
|
testutil.AssertFileContains(t, filepath.Join(htmlDestination, "report.html"), "<h1>Report</h1>")
|
||||||
if _, err := os.Stat(filepath.Join(htmlDestination, "report.md")); !os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(htmlDestination, "report.md")); !os.IsNotExist(err) {
|
||||||
t.Fatalf("html report.md stat error = %v, want not exist", err)
|
t.Fatalf("html report.md stat error = %v, want not exist", err)
|
||||||
}
|
}
|
||||||
@@ -649,10 +1228,10 @@ func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
|
|||||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &publishOutput}, provider); err != nil {
|
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &publishOutput}, provider); err != nil {
|
||||||
t.Fatalf("publish error = %v", err)
|
t.Fatalf("publish error = %v", err)
|
||||||
}
|
}
|
||||||
assertFakeFile(t, s3Destination, "report.md", "# Report\nSunny.\n")
|
testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nSunny.\n")
|
||||||
assertFakeFile(t, sshDestination, "summary.txt", "Summary\n")
|
testutil.AssertFakeFile(t, sshDestination, "summary.txt", "Summary\n")
|
||||||
assertFile(t, filepath.Join(s3ToLocalDestination, "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(s3ToLocalDestination, "report.md"), "# Report\nSunny.\n")
|
||||||
assertFile(t, filepath.Join(sshToLocalDestination, "summary.txt"), "Summary\n")
|
testutil.AssertFile(t, filepath.Join(sshToLocalDestination, "summary.txt"), "Summary\n")
|
||||||
|
|
||||||
var repeatOutput bytes.Buffer
|
var repeatOutput bytes.Buffer
|
||||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &repeatOutput}, provider); err != nil {
|
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &repeatOutput}, provider); err != nil {
|
||||||
@@ -668,10 +1247,10 @@ func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
|
|||||||
writeSourceBundle(t, localSourceRoot, "bundle", testBundleOptions{})
|
writeSourceBundle(t, localSourceRoot, "bundle", testBundleOptions{})
|
||||||
s3Destination := fake.New()
|
s3Destination := fake.New()
|
||||||
sshDestination := fake.New()
|
sshDestination := fake.New()
|
||||||
mustWriteFake(t, s3Destination, "bundle/old.txt", "old")
|
testutil.WriteFakeFile(t, s3Destination, "bundle/old.txt", "old")
|
||||||
mustWriteFake(t, s3Destination, "bundle-sibling/keep.txt", "keep")
|
testutil.WriteFakeFile(t, s3Destination, "bundle-sibling/keep.txt", "keep")
|
||||||
mustWriteFake(t, sshDestination, "bundle/old.txt", "old")
|
testutil.WriteFakeFile(t, sshDestination, "bundle/old.txt", "old")
|
||||||
mustWriteFake(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
testutil.WriteFakeFile(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
||||||
cfg := config.Config{Pipelines: []config.Pipeline{{
|
cfg := config.Config{Pipelines: []config.Pipeline{{
|
||||||
ID: "reports",
|
ID: "reports",
|
||||||
Source: config.Backend{Backend: config.BackendLocal, Path: localSourceRoot},
|
Source: config.Backend{Backend: config.BackendLocal, Path: localSourceRoot},
|
||||||
@@ -699,12 +1278,12 @@ func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
|
|||||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Force: true}, provider); err != nil {
|
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Force: true}, provider); err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
assertFakeFile(t, s3Destination, "bundle/report.md", "# Report\nSunny.\n")
|
testutil.AssertFakeFile(t, s3Destination, "bundle/report.md", "# Report\nSunny.\n")
|
||||||
assertFakeMissing(t, s3Destination, "bundle/old.txt")
|
testutil.AssertFakeMissing(t, s3Destination, "bundle/old.txt")
|
||||||
assertFakeFile(t, s3Destination, "bundle-sibling/keep.txt", "keep")
|
testutil.AssertFakeFile(t, s3Destination, "bundle-sibling/keep.txt", "keep")
|
||||||
assertFakeFile(t, sshDestination, "bundle/report.md", "# Report\nSunny.\n")
|
testutil.AssertFakeFile(t, sshDestination, "bundle/report.md", "# Report\nSunny.\n")
|
||||||
assertFakeMissing(t, sshDestination, "bundle/old.txt")
|
testutil.AssertFakeMissing(t, sshDestination, "bundle/old.txt")
|
||||||
assertFakeFile(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
testutil.AssertFakeFile(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunDryRunDoesNotWrite(t *testing.T) {
|
func TestRunDryRunDoesNotWrite(t *testing.T) {
|
||||||
@@ -727,6 +1306,7 @@ func TestRunDryRunDoesNotWrite(t *testing.T) {
|
|||||||
type testBundleOptions struct {
|
type testBundleOptions struct {
|
||||||
ID string
|
ID string
|
||||||
Created time.Time
|
Created time.Time
|
||||||
|
Files []testFile
|
||||||
ExtraFiles []testFile
|
ExtraFiles []testFile
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -737,6 +1317,13 @@ type testFile struct {
|
|||||||
|
|
||||||
func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptions) bundle.Manifest {
|
func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptions) bundle.Manifest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
var files []testutil.SourceFile
|
||||||
|
if opts.Files != nil {
|
||||||
|
files = make([]testutil.SourceFile, 0, len(opts.Files))
|
||||||
|
for _, file := range opts.Files {
|
||||||
|
files = append(files, testutil.SourceFile{Path: file.Path, Data: file.Data})
|
||||||
|
}
|
||||||
|
}
|
||||||
extraFiles := make([]testutil.SourceFile, 0, len(opts.ExtraFiles))
|
extraFiles := make([]testutil.SourceFile, 0, len(opts.ExtraFiles))
|
||||||
for _, file := range opts.ExtraFiles {
|
for _, file := range opts.ExtraFiles {
|
||||||
extraFiles = append(extraFiles, testutil.SourceFile{Path: file.Path, Data: file.Data})
|
extraFiles = append(extraFiles, testutil.SourceFile{Path: file.Path, Data: file.Data})
|
||||||
@@ -744,6 +1331,7 @@ func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptio
|
|||||||
return testutil.WriteSourceBundle(t, root, relative, testutil.BundleOptions{
|
return testutil.WriteSourceBundle(t, root, relative, testutil.BundleOptions{
|
||||||
ID: opts.ID,
|
ID: opts.ID,
|
||||||
Created: opts.Created,
|
Created: opts.Created,
|
||||||
|
Files: files,
|
||||||
ExtraFiles: extraFiles,
|
ExtraFiles: extraFiles,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -753,65 +1341,11 @@ func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
|||||||
return testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
return testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeLocalConfigWithPolicy(t *testing.T, sourceRoot, destinationRoot string, publishSource, publishHTML bool) string {
|
|
||||||
t.Helper()
|
|
||||||
transformConfig := ""
|
|
||||||
if publishHTML {
|
|
||||||
transformConfig = `
|
|
||||||
transform:
|
|
||||||
markdown_to_html:
|
|
||||||
enabled: true
|
|
||||||
mode: sidecar`
|
|
||||||
}
|
|
||||||
return writeConfigFile(t, `
|
|
||||||
pipelines:
|
|
||||||
- id: reports
|
|
||||||
source:
|
|
||||||
backend: local
|
|
||||||
path: `+sourceRoot+`
|
|
||||||
destinations:
|
|
||||||
- id: archive
|
|
||||||
backend: local
|
|
||||||
path: `+destinationRoot+`
|
|
||||||
publish:
|
|
||||||
source: `+fmt.Sprintf("%t", publishSource)+`
|
|
||||||
html: `+fmt.Sprintf("%t", publishHTML)+transformConfig+`
|
|
||||||
`)
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestination string) string {
|
func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestination string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
return testutil.WriteFanoutLocalConfig(t, sourceRoot, firstDestination, secondDestination)
|
return testutil.WriteFanoutLocalConfig(t, sourceRoot, firstDestination, secondDestination)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeMixedPolicyFanoutConfig(t *testing.T, sourceRoot, archiveDestination, htmlDestination string) string {
|
|
||||||
t.Helper()
|
|
||||||
return writeConfigFile(t, `
|
|
||||||
pipelines:
|
|
||||||
- id: reports
|
|
||||||
source:
|
|
||||||
backend: local
|
|
||||||
path: `+sourceRoot+`
|
|
||||||
destinations:
|
|
||||||
- id: archive
|
|
||||||
backend: local
|
|
||||||
path: `+archiveDestination+`
|
|
||||||
publish:
|
|
||||||
source: true
|
|
||||||
html: false
|
|
||||||
- id: html
|
|
||||||
backend: local
|
|
||||||
path: `+htmlDestination+`
|
|
||||||
publish:
|
|
||||||
source: false
|
|
||||||
html: true
|
|
||||||
transform:
|
|
||||||
markdown_to_html:
|
|
||||||
enabled: true
|
|
||||||
mode: sidecar
|
|
||||||
`)
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeConfigFile(t *testing.T, body string) string {
|
func writeConfigFile(t *testing.T, body string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join(t.TempDir(), "config.yml")
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
@@ -831,51 +1365,12 @@ func readStateFile(t *testing.T, path string) state.DistributorState {
|
|||||||
return testutil.ReadDestinationState(t, path)
|
return testutil.ReadDestinationState(t, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertFile(t *testing.T, path, want string) {
|
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
|
||||||
t.Helper()
|
byPath := make(map[string]state.OutputFile, len(outputs))
|
||||||
data, err := os.ReadFile(path)
|
for _, output := range outputs {
|
||||||
if err != nil {
|
byPath[output.Path] = output
|
||||||
t.Fatalf("read file %s: %v", path, err)
|
|
||||||
}
|
|
||||||
if got := string(data); got != want {
|
|
||||||
t.Fatalf("%s = %q, want %q", path, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertFileContains(t *testing.T, path, want string) {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read file %s: %v", path, err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), want) {
|
|
||||||
t.Fatalf("%s = %q, want substring %q", path, data, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertFakeFile(t *testing.T, backend *fake.Backend, path, want string) {
|
|
||||||
t.Helper()
|
|
||||||
data, err := backend.ReadFile(context.Background(), path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read fake file %s: %v", path, err)
|
|
||||||
}
|
|
||||||
if got := string(data); got != want {
|
|
||||||
t.Fatalf("%s = %q, want %q", path, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertFakeMissing(t *testing.T, backend *fake.Backend, path string) {
|
|
||||||
t.Helper()
|
|
||||||
if _, err := backend.Stat(context.Background(), path); !storage.IsNotFound(err) {
|
|
||||||
t.Fatalf("fake file %s stat error = %v, want not found", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustWriteFake(t *testing.T, backend *fake.Backend, path, data string) {
|
|
||||||
t.Helper()
|
|
||||||
if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
|
|
||||||
t.Fatalf("write fake file %s: %v", path, err)
|
|
||||||
}
|
}
|
||||||
|
return byPath
|
||||||
}
|
}
|
||||||
|
|
||||||
func crossBackendConfig(localSourceRoot, s3ToLocalDestination, sshToLocalDestination string) config.Config {
|
func crossBackendConfig(localSourceRoot, s3ToLocalDestination, sshToLocalDestination string) config.Config {
|
||||||
|
|||||||
47
internal/app/run_warnings.go
Normal file
47
internal/app/run_warnings.go
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func secretConflictWarnings(conflicts []config.SecretConflict) []OutputWarning {
|
||||||
|
warnings := make([]OutputWarning, 0, len(conflicts))
|
||||||
|
for _, conflict := range conflicts {
|
||||||
|
warnings = append(warnings, OutputWarning{
|
||||||
|
Message: fmt.Sprintf("secret %s ignored because the real environment already has that variable", conflict.Name),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
func sshWarnings(pipeline config.Pipeline) []OutputWarning {
|
||||||
|
var warnings []OutputWarning
|
||||||
|
if pipeline.Source.Backend == config.BackendSSH && pipeline.Source.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
||||||
|
warnings = append(warnings, OutputWarning{
|
||||||
|
Message: fmt.Sprintf("pipeline=%s source host_key_policy=off disables SSH host key checking", pipeline.ID),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, destination := range pipeline.Destinations {
|
||||||
|
if destination.Backend == config.BackendSSH && destination.SSH.HostKeyPolicy == config.HostKeyPolicyOff {
|
||||||
|
warnings = append(warnings, OutputWarning{
|
||||||
|
Message: fmt.Sprintf("pipeline=%s destination=%s host_key_policy=off disables SSH host key checking", pipeline.ID, destination.ID),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeWarnings(w io.Writer, warnings []OutputWarning) error {
|
||||||
|
if w == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, warning := range warnings {
|
||||||
|
if _, err := fmt.Fprintf(w, "Warning: %s\n", warning.Message); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
120
internal/app/source_select.go
Normal file
120
internal/app/source_select.go
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type sourceCommandOptions struct {
|
||||||
|
CommandName string
|
||||||
|
Path string
|
||||||
|
ConfigPath string
|
||||||
|
PipelineID string
|
||||||
|
BundlePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
type sourceSelection struct {
|
||||||
|
Bundles []bundle.Bundle
|
||||||
|
PipelineID string
|
||||||
|
SourceBackend string
|
||||||
|
ConfigMode bool
|
||||||
|
Warnings []OutputWarning
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectSourceBundles(ctx context.Context, options sourceCommandOptions, provider backendFactoryProvider) (sourceSelection, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return sourceSelection{}, err
|
||||||
|
}
|
||||||
|
if options.ConfigPath != "" {
|
||||||
|
cfg, err := config.LoadFile(options.ConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
return sourceSelection{}, err
|
||||||
|
}
|
||||||
|
return selectSourceBundlesFromConfig(ctx, cfg, options, provider)
|
||||||
|
}
|
||||||
|
if options.PipelineID != "" {
|
||||||
|
return sourceSelection{}, fmt.Errorf("configured source mode requires --config")
|
||||||
|
}
|
||||||
|
if options.BundlePath != "" {
|
||||||
|
return sourceSelection{}, fmt.Errorf("configured source mode requires --config")
|
||||||
|
}
|
||||||
|
if options.Path == "" {
|
||||||
|
return sourceSelection{}, fmt.Errorf("%s command requires a path", options.CommandName)
|
||||||
|
}
|
||||||
|
backend, err := newBackendFactory().openLocalPath(ctx, options.Path)
|
||||||
|
if err != nil {
|
||||||
|
return sourceSelection{}, err
|
||||||
|
}
|
||||||
|
defer closeBackend(backend)
|
||||||
|
bundles, err := bundle.Discover(ctx, backend, "")
|
||||||
|
if err != nil {
|
||||||
|
return sourceSelection{}, err
|
||||||
|
}
|
||||||
|
return sourceSelection{Bundles: bundles}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectSourceBundlesFromConfig(ctx context.Context, cfg config.Config, options sourceCommandOptions, provider backendFactoryProvider) (sourceSelection, error) {
|
||||||
|
if options.Path != "" {
|
||||||
|
return sourceSelection{}, fmt.Errorf("configured source mode does not accept a local path")
|
||||||
|
}
|
||||||
|
if options.PipelineID == "" {
|
||||||
|
return sourceSelection{}, fmt.Errorf("configured source mode requires --pipeline")
|
||||||
|
}
|
||||||
|
secretLoad, err := config.LoadSecretEnvironment(cfg.Secrets.Directory, nil)
|
||||||
|
if err != nil {
|
||||||
|
return sourceSelection{}, err
|
||||||
|
}
|
||||||
|
pipeline, ok := findPipeline(cfg, options.PipelineID)
|
||||||
|
if !ok {
|
||||||
|
return sourceSelection{}, fmt.Errorf("pipeline %q not found", options.PipelineID)
|
||||||
|
}
|
||||||
|
backends := provider(secretLoad.Environment)
|
||||||
|
sourceBackend, err := backends.openSource(ctx, pipeline.Source)
|
||||||
|
if err != nil {
|
||||||
|
return sourceSelection{}, fmt.Errorf("pipeline %s source backend %s: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||||
|
}
|
||||||
|
defer closeBackend(sourceBackend)
|
||||||
|
|
||||||
|
var bundles []bundle.Bundle
|
||||||
|
if options.BundlePath != "" {
|
||||||
|
sourceBundle, err := bundle.Validate(ctx, sourceBackend, options.BundlePath)
|
||||||
|
if err != nil {
|
||||||
|
return sourceSelection{}, fmt.Errorf("pipeline %s source backend %s bundle %s: %w", pipeline.ID, pipeline.Source.Backend, storage.DisplayPath(options.BundlePath), err)
|
||||||
|
}
|
||||||
|
bundles = []bundle.Bundle{sourceBundle}
|
||||||
|
} else {
|
||||||
|
bundles, err = bundle.Discover(ctx, sourceBackend, "")
|
||||||
|
if err != nil {
|
||||||
|
return sourceSelection{}, fmt.Errorf("pipeline %s source backend %s discover source bundles: %w", pipeline.ID, pipeline.Source.Backend, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sourceSelection{
|
||||||
|
Bundles: bundles,
|
||||||
|
PipelineID: pipeline.ID,
|
||||||
|
SourceBackend: pipeline.Source.Backend,
|
||||||
|
ConfigMode: true,
|
||||||
|
Warnings: append(secretConflictWarnings(secretLoad.Conflicts), sourceSSHWarnings(pipeline)...),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func findPipeline(cfg config.Config, id string) (config.Pipeline, bool) {
|
||||||
|
for _, pipeline := range cfg.Pipelines {
|
||||||
|
if pipeline.ID == id {
|
||||||
|
return pipeline, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return config.Pipeline{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func sourceSSHWarnings(pipeline config.Pipeline) []OutputWarning {
|
||||||
|
if pipeline.Source.Backend != config.BackendSSH || pipeline.Source.SSH.HostKeyPolicy != config.HostKeyPolicyOff {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []OutputWarning{{
|
||||||
|
Message: fmt.Sprintf("pipeline=%s source host_key_policy=off disables SSH host key checking", pipeline.ID),
|
||||||
|
}}
|
||||||
|
}
|
||||||
@@ -5,28 +5,97 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ValidateOptions struct {
|
type ValidateOptions struct {
|
||||||
Path string
|
Path string
|
||||||
Stdout io.Writer
|
ConfigPath string
|
||||||
|
PipelineID string
|
||||||
|
BundlePath string
|
||||||
|
Stdout io.Writer
|
||||||
|
OutputFormat OutputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
func Validate(ctx context.Context, options ValidateOptions) error {
|
func Validate(ctx context.Context, options ValidateOptions) error {
|
||||||
if options.Path == "" {
|
return validateWithBackendFactory(ctx, options, newBackendFactoryWithEnvironment)
|
||||||
return fmt.Errorf("validate command requires a path")
|
}
|
||||||
|
|
||||||
|
func validateWithBackendFactory(ctx context.Context, options ValidateOptions, provider backendFactoryProvider) error {
|
||||||
|
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
backend, err := newBackendFactory().openLocalPath(ctx, options.Path)
|
selection, err := selectSourceBundles(ctx, sourceCommandOptions{
|
||||||
|
CommandName: "validate",
|
||||||
|
Path: options.Path,
|
||||||
|
ConfigPath: options.ConfigPath,
|
||||||
|
PipelineID: options.PipelineID,
|
||||||
|
BundlePath: options.BundlePath,
|
||||||
|
}, provider)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bundles, err := bundle.Discover(ctx, backend, "")
|
return writeValidateResult(options, selection)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateConfigWithBackendFactory(ctx context.Context, cfg config.Config, options ValidateOptions, provider backendFactoryProvider) error {
|
||||||
|
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
selection, err := selectSourceBundlesFromConfig(ctx, cfg, sourceCommandOptions{
|
||||||
|
CommandName: "validate",
|
||||||
|
PipelineID: options.PipelineID,
|
||||||
|
BundlePath: options.BundlePath,
|
||||||
|
}, provider)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
return writeValidateResult(options, selection)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeValidateResult(options ValidateOptions, selection sourceSelection) error {
|
||||||
|
if IsJSONOutput(options.OutputFormat) {
|
||||||
|
return WriteJSONEnvelope(options.Stdout, "validate", true, selection.Warnings, validateResultFromSelection(selection), nil)
|
||||||
|
}
|
||||||
|
var err error
|
||||||
if options.Stdout != nil {
|
if options.Stdout != nil {
|
||||||
_, err = fmt.Fprintf(options.Stdout, "Validated %d bundle(s)\n", len(bundles))
|
if err := writeWarnings(options.Stdout, selection.Warnings); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if selection.ConfigMode {
|
||||||
|
_, err = fmt.Fprintf(options.Stdout, "Validated %d bundle(s) for pipeline %s source %s\n", len(selection.Bundles), selection.PipelineID, selection.SourceBackend)
|
||||||
|
} else {
|
||||||
|
_, err = fmt.Fprintf(options.Stdout, "Validated %d bundle(s)\n", len(selection.Bundles))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type validateResult struct {
|
||||||
|
PipelineID string `json:"pipeline_id,omitempty"`
|
||||||
|
SourceBackend string `json:"source_backend,omitempty"`
|
||||||
|
BundleCount int `json:"bundle_count"`
|
||||||
|
Bundles []validateBundleResult `json:"bundles"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type validateBundleResult struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateResultFromSelection(selection sourceSelection) validateResult {
|
||||||
|
result := validateResult{
|
||||||
|
PipelineID: selection.PipelineID,
|
||||||
|
SourceBackend: selection.SourceBackend,
|
||||||
|
BundleCount: len(selection.Bundles),
|
||||||
|
Bundles: make([]validateBundleResult, 0, len(selection.Bundles)),
|
||||||
|
}
|
||||||
|
for _, sourceBundle := range selection.Bundles {
|
||||||
|
result.Bundles = append(result.Bundles, validateBundleResult{
|
||||||
|
Path: storage.DisplayPath(sourceBundle.RootRelativePath),
|
||||||
|
ID: sourceBundle.Manifest.ID,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,9 +3,15 @@ package app
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestValidateLocalBundle(t *testing.T) {
|
func TestValidateLocalBundle(t *testing.T) {
|
||||||
@@ -31,9 +37,182 @@ func TestValidateExampleSourceBundle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateConfiguredLocalSource(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
err := Validate(context.Background(), ValidateOptions{
|
||||||
|
ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
PipelineID: "reports",
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() configured source error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := stdout.String(), "Validated 1 bundle(s) for pipeline reports source local\n"; got != want {
|
||||||
|
t.Fatalf("stdout = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateConfiguredSourceBundlePath(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "daily/one", testutil.BundleOptions{ID: "reports.one"})
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "daily/two", testutil.BundleOptions{ID: "reports.two"})
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
|
||||||
|
err := Validate(context.Background(), ValidateOptions{
|
||||||
|
ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
PipelineID: "reports",
|
||||||
|
BundlePath: "daily/two",
|
||||||
|
Stdout: &stdout,
|
||||||
|
OutputFormat: OutputFormatJSON,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() configured bundle error = %v", err)
|
||||||
|
}
|
||||||
|
result := decodeAppResult(t, stdout.String())
|
||||||
|
if result["pipeline_id"] != "reports" || result["source_backend"] != "local" || result["bundle_count"] != float64(1) {
|
||||||
|
t.Fatalf("result = %#v, want configured source summary", result)
|
||||||
|
}
|
||||||
|
bundles, ok := result["bundles"].([]any)
|
||||||
|
if !ok || len(bundles) != 1 {
|
||||||
|
t.Fatalf("bundles = %#v, want one bundle", result["bundles"])
|
||||||
|
}
|
||||||
|
sourceBundle, ok := bundles[0].(map[string]any)
|
||||||
|
if !ok || sourceBundle["path"] != "daily/two" || sourceBundle["id"] != "reports.two" {
|
||||||
|
t.Fatalf("bundle = %#v, want narrowed bundle", sourceBundle)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateConfiguredRemoteSourcesThroughStorageAbstraction(t *testing.T) {
|
||||||
|
s3Source := fake.New()
|
||||||
|
testutil.WriteFakeSourceBundle(t, s3Source, "", testutil.BundleOptions{ID: "reports.s3"})
|
||||||
|
sshSource := fake.New()
|
||||||
|
testutil.WriteFakeSourceBundle(t, sshSource, "daily", testutil.BundleOptions{ID: "reports.ssh"})
|
||||||
|
cfg := config.Config{Pipelines: []config.Pipeline{
|
||||||
|
{
|
||||||
|
ID: "s3-reports",
|
||||||
|
Source: config.Backend{
|
||||||
|
Backend: config.BackendS3,
|
||||||
|
Endpoint: "http://s3.test",
|
||||||
|
Bucket: "source-bucket",
|
||||||
|
},
|
||||||
|
Destinations: []config.Destination{{
|
||||||
|
ID: "archive",
|
||||||
|
Backend: config.BackendLocal,
|
||||||
|
Path: t.TempDir(),
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "ssh-reports",
|
||||||
|
Source: config.Backend{
|
||||||
|
Backend: config.BackendSSH,
|
||||||
|
Host: "ssh.test",
|
||||||
|
Path: "/source",
|
||||||
|
},
|
||||||
|
Destinations: []config.Destination{{
|
||||||
|
ID: "archive",
|
||||||
|
Backend: config.BackendLocal,
|
||||||
|
Path: t.TempDir(),
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
config.ApplyDefaults(&cfg)
|
||||||
|
provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
|
||||||
|
"s3:source-bucket": s3Source,
|
||||||
|
"ssh:/source": sshSource,
|
||||||
|
})
|
||||||
|
|
||||||
|
var s3Stdout bytes.Buffer
|
||||||
|
if err := validateConfigWithBackendFactory(context.Background(), cfg, ValidateOptions{
|
||||||
|
PipelineID: "s3-reports",
|
||||||
|
Stdout: &s3Stdout,
|
||||||
|
OutputFormat: OutputFormatJSON,
|
||||||
|
}, provider); err != nil {
|
||||||
|
t.Fatalf("validate s3 source error = %v", err)
|
||||||
|
}
|
||||||
|
s3Result := decodeAppResult(t, s3Stdout.String())
|
||||||
|
if s3Result["source_backend"] != "s3" || s3Result["bundle_count"] != float64(1) {
|
||||||
|
t.Fatalf("s3 result = %#v, want one s3 bundle", s3Result)
|
||||||
|
}
|
||||||
|
|
||||||
|
var sshStdout bytes.Buffer
|
||||||
|
if err := validateConfigWithBackendFactory(context.Background(), cfg, ValidateOptions{
|
||||||
|
PipelineID: "ssh-reports",
|
||||||
|
BundlePath: "daily",
|
||||||
|
Stdout: &sshStdout,
|
||||||
|
}, provider); err != nil {
|
||||||
|
t.Fatalf("validate ssh source error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(sshStdout.String(), "pipeline ssh-reports source ssh") {
|
||||||
|
t.Fatalf("ssh stdout = %q, want ssh source summary", sshStdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateConfiguredSourceLoadsSecretsBeforeOpeningBackend(t *testing.T) {
|
||||||
|
sourceRoot := filepath.Join(t.TempDir(), "missing-source")
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
configPath := writeConfigFile(t, `
|
||||||
|
secrets:
|
||||||
|
directory: `+filepath.Join(t.TempDir(), "missing-secrets")+`
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
`)
|
||||||
|
|
||||||
|
err := Validate(context.Background(), ValidateOptions{ConfigPath: configPath, PipelineID: "reports"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want secrets directory error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "load secrets directory") {
|
||||||
|
t.Fatalf("Validate() error = %v, want secrets directory error", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "missing-source") {
|
||||||
|
t.Fatalf("Validate() error = %v, opened source before loading secrets", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateConfiguredSourceRequiresPipeline(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
|
||||||
|
err := Validate(context.Background(), ValidateOptions{
|
||||||
|
ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
})
|
||||||
|
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "requires --pipeline") {
|
||||||
|
t.Fatalf("Validate() error = %v, want required pipeline", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateRequiresPath(t *testing.T) {
|
func TestValidateRequiresPath(t *testing.T) {
|
||||||
err := Validate(context.Background(), ValidateOptions{})
|
err := Validate(context.Background(), ValidateOptions{})
|
||||||
if err == nil || !strings.Contains(err.Error(), "requires a path") {
|
if err == nil || !strings.Contains(err.Error(), "requires a path") {
|
||||||
t.Fatalf("Validate() error = %v, want required path", err)
|
t.Fatalf("Validate() error = %v, want required path", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func decodeAppResult(t *testing.T, output string) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
var envelope map[string]any
|
||||||
|
if err := json.Unmarshal([]byte(output), &envelope); err != nil {
|
||||||
|
t.Fatalf("decode output: %v; output = %q", err, output)
|
||||||
|
}
|
||||||
|
result, ok := envelope["result"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("result = %#v, want object", envelope["result"])
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,49 +1,19 @@
|
|||||||
package bundle
|
package bundle
|
||||||
|
|
||||||
import (
|
import publicbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
|
||||||
|
|
||||||
func ValidateDigest(value string) error {
|
func ValidateDigest(value string) error {
|
||||||
if !digestPattern.MatchString(value) {
|
return publicbundle.ValidateDigest(value)
|
||||||
return fmt.Errorf("must be lowercase sha256:<64 hex>")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func FileDigest(data []byte) string {
|
func FileDigest(data []byte) string {
|
||||||
sum := sha256.Sum256(data)
|
return publicbundle.FileDigest(data)
|
||||||
return "sha256:" + hex.EncodeToString(sum[:])
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func BundleDigest(files []ManifestFile) string {
|
func BundleDigest(files []ManifestFile) string {
|
||||||
canonical := CanonicalFilePayload(files)
|
return publicbundle.BundleDigest(files)
|
||||||
sum := sha256.Sum256([]byte(canonical))
|
|
||||||
return "sha256:" + hex.EncodeToString(sum[:])
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func CanonicalFilePayload(files []ManifestFile) string {
|
func CanonicalFilePayload(files []ManifestFile) string {
|
||||||
var builder strings.Builder
|
return publicbundle.CanonicalFilePayload(files)
|
||||||
builder.WriteByte('[')
|
|
||||||
for index, file := range files {
|
|
||||||
if index > 0 {
|
|
||||||
builder.WriteByte(',')
|
|
||||||
}
|
|
||||||
builder.WriteString(`{"path":`)
|
|
||||||
builder.WriteString(strconv.Quote(file.Path))
|
|
||||||
builder.WriteString(`,"sha256":`)
|
|
||||||
builder.WriteString(strconv.Quote(file.SHA256))
|
|
||||||
builder.WriteString(`,"size":`)
|
|
||||||
builder.WriteString(strconv.FormatInt(file.Size, 10))
|
|
||||||
builder.WriteByte('}')
|
|
||||||
}
|
|
||||||
builder.WriteByte(']')
|
|
||||||
return builder.String()
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,110 +1,32 @@
|
|||||||
package bundle
|
package bundle
|
||||||
|
|
||||||
import (
|
import publicbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const ManifestName = "manifest.json"
|
const ManifestName = publicbundle.ManifestName
|
||||||
|
|
||||||
type Manifest struct {
|
const SchemaVersion = publicbundle.SchemaVersion
|
||||||
SchemaVersion int `json:"schema_version"`
|
|
||||||
ID string `json:"id"`
|
|
||||||
Digest string `json:"digest"`
|
|
||||||
Created time.Time `json:"created"`
|
|
||||||
Files []ManifestFile `json:"files"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ManifestFile struct {
|
type Manifest = publicbundle.Manifest
|
||||||
Path string `json:"path"`
|
|
||||||
SHA256 string `json:"sha256"`
|
type ManifestFile = publicbundle.ManifestFile
|
||||||
Size int64 `json:"size"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Bundle struct {
|
type Bundle struct {
|
||||||
RootRelativePath string
|
RootRelativePath string
|
||||||
Manifest Manifest
|
Manifest Manifest
|
||||||
}
|
}
|
||||||
|
|
||||||
type rawManifest struct {
|
|
||||||
SchemaVersion *int `json:"schema_version"`
|
|
||||||
ID *string `json:"id"`
|
|
||||||
Digest *string `json:"digest"`
|
|
||||||
Created *string `json:"created"`
|
|
||||||
Files []rawManifestFile `json:"files"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawManifestFile struct {
|
|
||||||
Path *string `json:"path"`
|
|
||||||
SHA256 *string `json:"sha256"`
|
|
||||||
Size *int64 `json:"size"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseManifest(data []byte) (Manifest, error) {
|
func ParseManifest(data []byte) (Manifest, error) {
|
||||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
return publicbundle.ParseManifest(data)
|
||||||
var raw rawManifest
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return Manifest{}, fmt.Errorf("parse manifest: %w", err)
|
|
||||||
}
|
|
||||||
var extra any
|
|
||||||
if err := decoder.Decode(&extra); err != io.EOF {
|
|
||||||
return Manifest{}, fmt.Errorf("parse manifest: trailing data")
|
|
||||||
}
|
|
||||||
|
|
||||||
var manifest Manifest
|
|
||||||
if raw.SchemaVersion == nil {
|
|
||||||
return Manifest{}, fmt.Errorf("manifest schema_version is required")
|
|
||||||
}
|
|
||||||
manifest.SchemaVersion = *raw.SchemaVersion
|
|
||||||
if raw.ID == nil || *raw.ID == "" {
|
|
||||||
return Manifest{}, fmt.Errorf("manifest id is required")
|
|
||||||
}
|
|
||||||
manifest.ID = *raw.ID
|
|
||||||
if raw.Digest == nil || *raw.Digest == "" {
|
|
||||||
return Manifest{}, fmt.Errorf("manifest digest is required")
|
|
||||||
}
|
|
||||||
manifest.Digest = *raw.Digest
|
|
||||||
if raw.Created == nil || *raw.Created == "" {
|
|
||||||
return Manifest{}, fmt.Errorf("manifest created is required")
|
|
||||||
}
|
|
||||||
created, err := time.Parse(time.RFC3339, *raw.Created)
|
|
||||||
if err != nil {
|
|
||||||
return Manifest{}, fmt.Errorf("manifest created must be RFC3339: %w", err)
|
|
||||||
}
|
|
||||||
manifest.Created = created
|
|
||||||
if len(raw.Files) == 0 {
|
|
||||||
return Manifest{}, fmt.Errorf("manifest files is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
for index, rawFile := range raw.Files {
|
|
||||||
file, err := parseManifestFile(index, rawFile)
|
|
||||||
if err != nil {
|
|
||||||
return Manifest{}, err
|
|
||||||
}
|
|
||||||
manifest.Files = append(manifest.Files, file)
|
|
||||||
}
|
|
||||||
if err := ValidateManifest(manifest); err != nil {
|
|
||||||
return Manifest{}, fmt.Errorf("manifest %w", err)
|
|
||||||
}
|
|
||||||
return manifest, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseManifestFile(index int, raw rawManifestFile) (ManifestFile, error) {
|
func MarshalManifest(manifest Manifest) ([]byte, error) {
|
||||||
if raw.Path == nil || *raw.Path == "" {
|
return publicbundle.MarshalManifest(manifest)
|
||||||
return ManifestFile{}, fmt.Errorf("manifest files[%d].path is required", index)
|
}
|
||||||
}
|
|
||||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
func ValidateManifest(manifest Manifest) error {
|
||||||
return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256 is required", index)
|
return publicbundle.ValidateManifest(manifest)
|
||||||
}
|
}
|
||||||
if raw.Size == nil {
|
|
||||||
return ManifestFile{}, fmt.Errorf("manifest files[%d].size is required", index)
|
func ValidateSourcePath(path string) error {
|
||||||
}
|
return publicbundle.ValidateSourcePath(path)
|
||||||
return ManifestFile{
|
|
||||||
Path: *raw.Path,
|
|
||||||
SHA256: *raw.SHA256,
|
|
||||||
Size: *raw.Size,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,55 +7,6 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ValidateSourcePath(path string) error {
|
|
||||||
if err := storage.ValidatePath(path); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
switch path {
|
|
||||||
case ManifestName, storage.StateFileName:
|
|
||||||
return fmt.Errorf("%q is reserved", path)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ValidateManifest(manifest Manifest) error {
|
|
||||||
if manifest.SchemaVersion != 1 {
|
|
||||||
return fmt.Errorf("schema_version must be 1")
|
|
||||||
}
|
|
||||||
if manifest.ID == "" {
|
|
||||||
return fmt.Errorf("id is required")
|
|
||||||
}
|
|
||||||
if err := ValidateDigest(manifest.Digest); err != nil {
|
|
||||||
return fmt.Errorf("digest: %w", err)
|
|
||||||
}
|
|
||||||
if manifest.Created.IsZero() {
|
|
||||||
return fmt.Errorf("created is required")
|
|
||||||
}
|
|
||||||
if len(manifest.Files) == 0 {
|
|
||||||
return fmt.Errorf("files is required")
|
|
||||||
}
|
|
||||||
seen := make(map[string]struct{}, len(manifest.Files))
|
|
||||||
for index, file := range manifest.Files {
|
|
||||||
if err := ValidateSourcePath(file.Path); err != nil {
|
|
||||||
return fmt.Errorf("files[%d].path: %w", index, err)
|
|
||||||
}
|
|
||||||
if err := ValidateDigest(file.SHA256); err != nil {
|
|
||||||
return fmt.Errorf("files[%d].sha256: %w", index, err)
|
|
||||||
}
|
|
||||||
if file.Size < 0 {
|
|
||||||
return fmt.Errorf("files[%d].size must be non-negative", index)
|
|
||||||
}
|
|
||||||
if _, exists := seen[file.Path]; exists {
|
|
||||||
return fmt.Errorf("files[%d].path duplicates %q", index, file.Path)
|
|
||||||
}
|
|
||||||
seen[file.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
if actual := BundleDigest(manifest.Files); actual != manifest.Digest {
|
|
||||||
return fmt.Errorf("digest mismatch: got %s want %s", actual, manifest.Digest)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func Validate(ctx context.Context, backend storage.Backend, bundleRoot string) (Bundle, error) {
|
func Validate(ctx context.Context, backend storage.Backend, bundleRoot string) (Bundle, error) {
|
||||||
return validateAt(ctx, backend, bundleRoot, bundleRoot)
|
return validateAt(ctx, backend, bundleRoot, bundleRoot)
|
||||||
}
|
}
|
||||||
|
|||||||
22
internal/cli/format.go
Normal file
22
internal/cli/format.go
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
func addFormatFlag(flags *flag.FlagSet) *string {
|
||||||
|
return flags.String("format", string(app.OutputFormatText), "output format: text or json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseOutputFormat(stderr io.Writer, command, raw string) (app.OutputFormat, bool) {
|
||||||
|
format := app.OutputFormat(raw)
|
||||||
|
if err := app.ValidateOutputFormat(format); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "%s: %s --format: %s\n", app.Name, command, err)
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return app.NormalizeOutputFormat(format), true
|
||||||
|
}
|
||||||
@@ -13,11 +13,18 @@ func inspectCommand(ctx context.Context, args []string, stdout, stderr io.Writer
|
|||||||
printInspectHelp(stdout)
|
printInspectHelp(stdout)
|
||||||
return exitOK
|
return exitOK
|
||||||
}
|
}
|
||||||
path, ok := parseOptionalPathArg(stderr, "inspect", args)
|
parsed, ok := parseSourceDiagnosticArgs(stderr, "inspect", args)
|
||||||
if !ok {
|
if !ok {
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
if err := app.Inspect(ctx, app.InspectOptions{Path: path, Stdout: stdout}); err != nil {
|
if err := app.Inspect(ctx, app.InspectOptions{
|
||||||
|
Path: parsed.Path,
|
||||||
|
ConfigPath: parsed.ConfigPath,
|
||||||
|
PipelineID: parsed.PipelineID,
|
||||||
|
BundlePath: parsed.BundlePath,
|
||||||
|
Stdout: stdout,
|
||||||
|
OutputFormat: parsed.OutputFormat,
|
||||||
|
}); err != nil {
|
||||||
return fail(stderr, err)
|
return fail(stderr, err)
|
||||||
}
|
}
|
||||||
return exitOK
|
return exitOK
|
||||||
@@ -25,8 +32,16 @@ func inspectCommand(ctx context.Context, args []string, stdout, stderr io.Writer
|
|||||||
|
|
||||||
func printInspectHelp(w io.Writer) {
|
func printInspectHelp(w io.Writer) {
|
||||||
fmt.Fprint(w, `Usage:
|
fmt.Fprint(w, `Usage:
|
||||||
distributor inspect <path>
|
distributor inspect [--format text|json] <path>
|
||||||
|
distributor inspect --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
|
||||||
|
|
||||||
Print a normalized summary of local source bundles.
|
Options:
|
||||||
|
--config <path> Path to config file for configured source inspection
|
||||||
|
--pipeline <id> Pipeline id to inspect in config mode
|
||||||
|
--bundle <path> Source-root-relative bundle path to inspect
|
||||||
|
--format text|json Output format
|
||||||
|
|
||||||
|
Print a normalized summary of local source bundles or a configured pipeline
|
||||||
|
source.
|
||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|||||||
139
internal/cli/manifest.go
Normal file
139
internal/cli/manifest.go
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
func manifestCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||||
|
if len(args) == 0 || args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
|
||||||
|
printManifestHelp(stdout)
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "create":
|
||||||
|
return manifestCreateCommand(ctx, args[1:], stdout, stderr)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(stderr, "%s: manifest unknown command %q\n\n", app.Name, args[0])
|
||||||
|
printManifestHelp(stderr)
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestCreateCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||||
|
if hasHelp(args) {
|
||||||
|
printManifestCreateHelp(stdout)
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
flags := flag.NewFlagSet("manifest create", flag.ContinueOnError)
|
||||||
|
flags.SetOutput(stderr)
|
||||||
|
id := flags.String("id", "", "source bundle id")
|
||||||
|
created := flags.String("created", "", "source created timestamp")
|
||||||
|
overwrite := flags.Bool("overwrite", false, "replace an existing manifest.json")
|
||||||
|
formatFlag := addFormatFlag(flags)
|
||||||
|
var files repeatedFlag
|
||||||
|
flags.Var(&files, "file", "bundle-relative file to include")
|
||||||
|
flagArgs, positionalArgs, ok := splitManifestCreateArgs(stderr, args)
|
||||||
|
if !ok {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if err := flags.Parse(flagArgs); err != nil {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if len(positionalArgs) != 1 {
|
||||||
|
fmt.Fprintf(stderr, "%s: manifest create requires exactly one bundle path\n", app.Name)
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
format, ok := parseOutputFormat(stderr, "manifest create", *formatFlag)
|
||||||
|
if !ok {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
err := app.ManifestCreate(ctx, app.ManifestCreateOptions{
|
||||||
|
Root: positionalArgs[0],
|
||||||
|
ID: *id,
|
||||||
|
Created: *created,
|
||||||
|
Files: []string(files),
|
||||||
|
Overwrite: *overwrite,
|
||||||
|
Stdout: stdout,
|
||||||
|
OutputFormat: format,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fail(stderr, err)
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitManifestCreateArgs(stderr io.Writer, args []string) ([]string, []string, bool) {
|
||||||
|
var flagArgs []string
|
||||||
|
var positionalArgs []string
|
||||||
|
for index := 0; index < len(args); index++ {
|
||||||
|
arg := args[index]
|
||||||
|
switch arg {
|
||||||
|
case "--overwrite":
|
||||||
|
flagArgs = append(flagArgs, arg)
|
||||||
|
case "--id", "--created", "--file", "--format":
|
||||||
|
if index+1 >= len(args) {
|
||||||
|
fmt.Fprintf(stderr, "%s: manifest create %s requires a value\n", app.Name, arg)
|
||||||
|
return nil, nil, false
|
||||||
|
}
|
||||||
|
flagArgs = append(flagArgs, arg, args[index+1])
|
||||||
|
index++
|
||||||
|
default:
|
||||||
|
if strings.HasPrefix(arg, "--id=") ||
|
||||||
|
strings.HasPrefix(arg, "--created=") ||
|
||||||
|
strings.HasPrefix(arg, "--file=") ||
|
||||||
|
strings.HasPrefix(arg, "--format=") {
|
||||||
|
flagArgs = append(flagArgs, arg)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(arg, "-") {
|
||||||
|
flagArgs = append(flagArgs, arg)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
positionalArgs = append(positionalArgs, arg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return flagArgs, positionalArgs, true
|
||||||
|
}
|
||||||
|
|
||||||
|
type repeatedFlag []string
|
||||||
|
|
||||||
|
func (f *repeatedFlag) String() string {
|
||||||
|
return fmt.Sprint([]string(*f))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *repeatedFlag) Set(value string) error {
|
||||||
|
*f = append(*f, value)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func printManifestHelp(w io.Writer) {
|
||||||
|
fmt.Fprint(w, `Usage:
|
||||||
|
distributor manifest <command> [options]
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
create Create a source bundle manifest
|
||||||
|
|
||||||
|
Use "distributor manifest <command> --help" for command-specific help.
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func printManifestCreateHelp(w io.Writer) {
|
||||||
|
fmt.Fprint(w, `Usage:
|
||||||
|
distributor manifest create <bundle-path> --id <bundle-id> [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--id <bundle-id> Source bundle id
|
||||||
|
--file <path> Bundle-relative file to include; repeatable
|
||||||
|
--created <time> RFC3339 source created timestamp
|
||||||
|
--overwrite Replace an existing manifest.json
|
||||||
|
--format text|json Output format
|
||||||
|
|
||||||
|
Create manifest.json for a local source bundle directory.
|
||||||
|
`)
|
||||||
|
}
|
||||||
@@ -33,6 +33,8 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
|||||||
return validateCommand(ctx, args[1:], stdout, stderr)
|
return validateCommand(ctx, args[1:], stdout, stderr)
|
||||||
case "inspect":
|
case "inspect":
|
||||||
return inspectCommand(ctx, args[1:], stdout, stderr)
|
return inspectCommand(ctx, args[1:], stdout, stderr)
|
||||||
|
case "manifest":
|
||||||
|
return manifestCommand(ctx, args[1:], stdout, stderr)
|
||||||
default:
|
default:
|
||||||
fmt.Fprintf(stderr, "%s: unknown command %q\n\n", app.Name, args[0])
|
fmt.Fprintf(stderr, "%s: unknown command %q\n\n", app.Name, args[0])
|
||||||
printRootHelp(stderr)
|
printRootHelp(stderr)
|
||||||
@@ -51,6 +53,7 @@ Commands:
|
|||||||
run Run configured distribution pipelines
|
run Run configured distribution pipelines
|
||||||
validate Validate a source bundle or bundle tree
|
validate Validate a source bundle or bundle tree
|
||||||
inspect Inspect bundles or distributor state
|
inspect Inspect bundles or distributor state
|
||||||
|
manifest Create source bundle manifests
|
||||||
|
|
||||||
Use "%s <command> --help" for command-specific help.
|
Use "%s <command> --help" for command-specific help.
|
||||||
`, app.Name, app.Name, app.Name)
|
`, app.Name, app.Name, app.Name)
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -10,8 +13,32 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||||
|
producerbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func decodeEnvelope(t *testing.T, stdout *bytes.Buffer) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
decoder := json.NewDecoder(strings.NewReader(stdout.String()))
|
||||||
|
var envelope map[string]any
|
||||||
|
if err := decoder.Decode(&envelope); err != nil {
|
||||||
|
t.Fatalf("decode JSON envelope: %v; stdout = %q", err, stdout.String())
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := decoder.Decode(&extra); err != io.EOF {
|
||||||
|
t.Fatalf("stdout contains more than one JSON document: %q", stdout.String())
|
||||||
|
}
|
||||||
|
return envelope
|
||||||
|
}
|
||||||
|
|
||||||
|
func envelopeResult(t *testing.T, envelope map[string]any) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
result, ok := envelope["result"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("result = %#v, want object", envelope["result"])
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRootHelp(t *testing.T) {
|
func TestExecuteRootHelp(t *testing.T) {
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
@@ -44,6 +71,43 @@ func TestExecuteVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteVersionJSON(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"version", "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
if envelope["command"] != "version" || envelope["ok"] != true {
|
||||||
|
t.Fatalf("envelope = %#v, want version ok", envelope)
|
||||||
|
}
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
if result["application"] != "distributor" || result["version"] != "dev" {
|
||||||
|
t.Fatalf("result = %#v, want application/version", result)
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRejectsInvalidFormat(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"version", "--format", "xml"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitUsage {
|
||||||
|
t.Fatalf("exit code = %d, want %d", code, exitUsage)
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "format must be text or json") {
|
||||||
|
t.Fatalf("stderr = %q, want invalid format error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteValidate(t *testing.T) {
|
func TestExecuteValidate(t *testing.T) {
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
@@ -57,6 +121,63 @@ func TestExecuteValidate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteValidateJSON(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"validate", "--format", "json", filepath.Join("..", "bundle", "testdata", "valid_bundle")}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
if envelope["command"] != "validate" || envelope["ok"] != true {
|
||||||
|
t.Fatalf("envelope = %#v, want validate ok", envelope)
|
||||||
|
}
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
if result["bundle_count"] != float64(1) {
|
||||||
|
t.Fatalf("result = %#v, want one bundle", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteValidateConfiguredSource(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"validate", "--config", configPath, "--pipeline", "reports"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
if got, want := stdout.String(), "Validated 1 bundle(s) for pipeline reports source local\n"; got != want {
|
||||||
|
t.Fatalf("stdout = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteValidateConfiguredSourceJSON(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "daily", testutil.BundleOptions{ID: "reports.daily"})
|
||||||
|
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"validate", "--config", configPath, "--pipeline", "reports", "--bundle", "daily", "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
if envelope["command"] != "validate" || envelope["ok"] != true {
|
||||||
|
t.Fatalf("envelope = %#v, want validate ok", envelope)
|
||||||
|
}
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
if result["pipeline_id"] != "reports" || result["source_backend"] != "local" || result["bundle_count"] != float64(1) {
|
||||||
|
t.Fatalf("result = %#v, want configured source metadata", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteValidateArgs(t *testing.T) {
|
func TestExecuteValidateArgs(t *testing.T) {
|
||||||
validPath := filepath.Join("..", "bundle", "testdata", "valid_bundle")
|
validPath := filepath.Join("..", "bundle", "testdata", "valid_bundle")
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -84,6 +205,36 @@ func TestExecuteValidateArgs(t *testing.T) {
|
|||||||
wantCode: exitUsage,
|
wantCode: exitUsage,
|
||||||
wantStderr: "accepts at most one path",
|
wantStderr: "accepts at most one path",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "path plus config",
|
||||||
|
args: []string{"validate", "--config", "config.yml", "--pipeline", "reports", validPath},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "does not accept a local path",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "pipeline without config",
|
||||||
|
args: []string{"validate", "--pipeline", "reports"},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "requires --config",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bundle without config",
|
||||||
|
args: []string{"validate", "--bundle", "daily"},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "requires --config",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "config without pipeline",
|
||||||
|
args: []string{"validate", "--config", "config.yml"},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "requires --pipeline",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid format",
|
||||||
|
args: []string{"validate", "--format", "xml", validPath},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "format must be text or json",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
@@ -115,6 +266,56 @@ func TestExecuteInspect(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteInspectJSON(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"inspect", "--format", "json", filepath.Join("..", "bundle", "testdata", "valid_bundle")}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
if envelope["command"] != "inspect" || envelope["ok"] != true {
|
||||||
|
t.Fatalf("envelope = %#v, want inspect ok", envelope)
|
||||||
|
}
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
bundles, ok := result["bundles"].([]any)
|
||||||
|
if !ok || len(bundles) != 1 {
|
||||||
|
t.Fatalf("bundles = %#v, want one bundle", result["bundles"])
|
||||||
|
}
|
||||||
|
bundle, ok := bundles[0].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("bundle = %#v, want object", bundles[0])
|
||||||
|
}
|
||||||
|
if bundle["id"] != "weather.daily.brentwood.2026-05-30" || bundle["file_count"] != float64(2) || bundle["total_size"] != float64(24) {
|
||||||
|
t.Fatalf("bundle = %#v, want normalized metadata", bundle)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteInspectConfiguredSource(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "daily", testutil.BundleOptions{ID: "reports.daily"})
|
||||||
|
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"inspect", "--config", configPath, "--pipeline", "reports"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"Pipeline: reports",
|
||||||
|
"Source: local",
|
||||||
|
"path=daily",
|
||||||
|
"id=reports.daily",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(stdout.String(), want) {
|
||||||
|
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteInspectArgs(t *testing.T) {
|
func TestExecuteInspectArgs(t *testing.T) {
|
||||||
validPath := filepath.Join("..", "bundle", "testdata", "valid_bundle")
|
validPath := filepath.Join("..", "bundle", "testdata", "valid_bundle")
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -142,6 +343,36 @@ func TestExecuteInspectArgs(t *testing.T) {
|
|||||||
wantCode: exitUsage,
|
wantCode: exitUsage,
|
||||||
wantStderr: "accepts at most one path",
|
wantStderr: "accepts at most one path",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "path plus config",
|
||||||
|
args: []string{"inspect", "--config", "config.yml", "--pipeline", "reports", validPath},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "does not accept a local path",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "pipeline without config",
|
||||||
|
args: []string{"inspect", "--pipeline", "reports"},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "requires --config",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bundle without config",
|
||||||
|
args: []string{"inspect", "--bundle", "daily"},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "requires --config",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "config without pipeline",
|
||||||
|
args: []string{"inspect", "--config", "config.yml"},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "requires --pipeline",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid format",
|
||||||
|
args: []string{"inspect", "--format", "xml", validPath},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "format must be text or json",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
@@ -160,6 +391,206 @@ func TestExecuteInspectArgs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteManifestCreateExplicitFiles(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeCLIFile(t, root, "b.txt", "bravo")
|
||||||
|
writeCLIFile(t, root, "nested/a.txt", "alpha")
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{
|
||||||
|
"manifest", "create", root,
|
||||||
|
"--id", "reports.explicit",
|
||||||
|
"--created", "2026-06-01T11:00:00Z",
|
||||||
|
"--file", "b.txt",
|
||||||
|
"--file", "nested/a.txt",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"created manifest.json",
|
||||||
|
"bundle: reports.explicit",
|
||||||
|
"files: 2",
|
||||||
|
"digest: sha256:",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(stdout.String(), want) {
|
||||||
|
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
manifest, err := producerbundle.LoadManifest(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := manifestPaths(manifest), []string{"b.txt", "nested/a.txt"}; !equalStrings(got, want) {
|
||||||
|
t.Fatalf("manifest paths = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
if err := producerbundle.ValidateBundle(root, manifest); err != nil {
|
||||||
|
t.Fatalf("ValidateBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
var validateStdout, validateStderr bytes.Buffer
|
||||||
|
validateCode := Execute(context.Background(), []string{"validate", root}, &validateStdout, &validateStderr)
|
||||||
|
if validateCode != exitOK {
|
||||||
|
t.Fatalf("validate exit code = %d, want %d; stderr = %q", validateCode, exitOK, validateStderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteManifestCreateScansBundle(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeCLIFile(t, root, "z.txt", "zulu")
|
||||||
|
writeCLIFile(t, root, ".env", "dotfile")
|
||||||
|
writeCLIFile(t, root, "nested/report.md", "# Report\n")
|
||||||
|
writeCLIFile(t, root, storage.StateFileName, "destination state")
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.scan"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
manifest, err := producerbundle.LoadManifest(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := manifestPaths(manifest), []string{".env", "nested/report.md", "z.txt"}; !equalStrings(got, want) {
|
||||||
|
t.Fatalf("manifest paths = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteManifestCreateJSON(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeCLIFile(t, root, "report.md", "# Report\n")
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.json", "--file", "report.md", "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
if envelope["command"] != "manifest create" || envelope["ok"] != true {
|
||||||
|
t.Fatalf("envelope = %#v, want manifest create ok", envelope)
|
||||||
|
}
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
if result["id"] != "reports.json" || result["file_count"] != float64(1) {
|
||||||
|
t.Fatalf("result = %#v, want manifest summary", result)
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteManifestCreateOverwrite(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeCLIFile(t, root, "report.md", "old\n")
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.old", "--file", "report.md"}, &stdout, &stderr)
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("initial exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
writeCLIFile(t, root, "report.md", "new\n")
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.new", "--file", "report.md"}, &stdout, &stderr)
|
||||||
|
if code != exitError {
|
||||||
|
t.Fatalf("overwrite exit code = %d, want %d", code, exitError)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "write manifest") {
|
||||||
|
t.Fatalf("stderr = %q, want write manifest error", stderr.String())
|
||||||
|
}
|
||||||
|
manifest, err := producerbundle.LoadManifest(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if manifest.ID != "reports.old" {
|
||||||
|
t.Fatalf("manifest id = %q, want reports.old", manifest.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.new", "--file", "report.md", "--overwrite"}, &stdout, &stderr)
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("overwrite exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
manifest, err = producerbundle.LoadManifest(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if manifest.ID != "reports.new" {
|
||||||
|
t.Fatalf("manifest id = %q, want reports.new", manifest.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteManifestCreateRejectsSymlink(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeCLIFile(t, root, "target.md", "# Report\n")
|
||||||
|
if err := os.Symlink("target.md", filepath.Join(root, "link.md")); err != nil {
|
||||||
|
t.Skipf("symlink unavailable: %v", err)
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"manifest", "create", root, "--id", "reports.link", "--file", "link.md"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitError {
|
||||||
|
t.Fatalf("exit code = %d, want %d", code, exitError)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "regular file") {
|
||||||
|
t.Fatalf("stderr = %q, want regular file error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteManifestCreateArgs(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeCLIFile(t, root, "report.md", "# Report\n")
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
wantCode int
|
||||||
|
wantStderr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing path",
|
||||||
|
args: []string{"manifest", "create", "--id", "reports.missing"},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "requires exactly one bundle path",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing id",
|
||||||
|
args: []string{"manifest", "create", root},
|
||||||
|
wantCode: exitError,
|
||||||
|
wantStderr: "requires --id",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bad created",
|
||||||
|
args: []string{"manifest", "create", root, "--id", "reports.bad", "--created", "June 1"},
|
||||||
|
wantCode: exitError,
|
||||||
|
wantStderr: "created must be RFC3339",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bad format",
|
||||||
|
args: []string{"manifest", "create", root, "--id", "reports.bad", "--format", "xml"},
|
||||||
|
wantCode: exitUsage,
|
||||||
|
wantStderr: "format must be text or json",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := Execute(context.Background(), tt.args, &stdout, &stderr)
|
||||||
|
if code != tt.wantCode {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, tt.wantCode, stderr.String())
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), tt.wantStderr) {
|
||||||
|
t.Fatalf("stderr = %q, want substring %q", stderr.String(), tt.wantStderr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRunDryRun(t *testing.T) {
|
func TestExecuteRunDryRun(t *testing.T) {
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
@@ -180,6 +611,269 @@ func TestExecuteRunDryRun(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunJSONDryRun(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, t.TempDir())
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run", "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
if envelope["command"] != "run" || envelope["ok"] != true {
|
||||||
|
t.Fatalf("envelope = %#v, want run ok", envelope)
|
||||||
|
}
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
if result["dry_run"] != true {
|
||||||
|
t.Fatalf("result = %#v, want dry_run true", result)
|
||||||
|
}
|
||||||
|
actions, ok := result["actions"].([]any)
|
||||||
|
if !ok || len(actions) != 1 {
|
||||||
|
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||||
|
}
|
||||||
|
action, ok := actions[0].(map[string]any)
|
||||||
|
if !ok || action["action"] != "publish_new" {
|
||||||
|
t.Fatalf("action = %#v, want publish_new", actions[0])
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunJSONDryRunReportsFixedPathMapping(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte(`
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: latest
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
path_mapping:
|
||||||
|
mode: fixed
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run", "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
warnings, ok := envelope["warnings"].([]any)
|
||||||
|
if !ok || len(warnings) != 1 {
|
||||||
|
t.Fatalf("warnings = %#v, want one fixed-path warning", envelope["warnings"])
|
||||||
|
}
|
||||||
|
warning, ok := warnings[0].(map[string]any)
|
||||||
|
if !ok || !strings.Contains(fmt.Sprint(warning["message"]), "path_mapping=fixed candidates=1 selected_bundle=.") {
|
||||||
|
t.Fatalf("warning = %#v, want fixed-path selection warning", warnings[0])
|
||||||
|
}
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
actions, ok := result["actions"].([]any)
|
||||||
|
if !ok || len(actions) != 1 {
|
||||||
|
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||||
|
}
|
||||||
|
action, ok := actions[0].(map[string]any)
|
||||||
|
if !ok || action["path_mapping"] != "fixed" || action["destination_path"] != "." || action["action"] != "publish_new" {
|
||||||
|
t.Fatalf("action = %#v, want fixed publish_new at root", actions[0])
|
||||||
|
}
|
||||||
|
summary, ok := result["summary"].(map[string]any)
|
||||||
|
if !ok || summary["fixed_path"] != float64(1) {
|
||||||
|
t.Fatalf("summary = %#v, want fixed_path 1", result["summary"])
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunJSONDryRunReportsLinks(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte(`
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: web
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
links:
|
||||||
|
base_url: https://reports.example.com/archive
|
||||||
|
primary: source
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run", "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
actions, ok := result["actions"].([]any)
|
||||||
|
if !ok || len(actions) != 1 {
|
||||||
|
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||||
|
}
|
||||||
|
action, ok := actions[0].(map[string]any)
|
||||||
|
if !ok || action["primary_url"] != "https://reports.example.com/archive/report.md" {
|
||||||
|
t.Fatalf("action = %#v, want primary URL", actions[0])
|
||||||
|
}
|
||||||
|
outputs, ok := action["outputs"].([]any)
|
||||||
|
if !ok || len(outputs) != 2 {
|
||||||
|
t.Fatalf("outputs = %#v, want two outputs", action["outputs"])
|
||||||
|
}
|
||||||
|
output, ok := outputs[0].(map[string]any)
|
||||||
|
if !ok || output["url"] != "https://reports.example.com/archive/report.md" {
|
||||||
|
t.Fatalf("output = %#v, want output URL", outputs[0])
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunJSONWarningsAreStructured(t *testing.T) {
|
||||||
|
name := "DISTRIBUTOR_TEST_CLI_JSON_SECRET"
|
||||||
|
t.Setenv(name, "process-value")
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
secretsRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
if err := os.WriteFile(filepath.Join(secretsRoot, name), []byte("secret-value\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write secret: %v", err)
|
||||||
|
}
|
||||||
|
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte(`
|
||||||
|
secrets:
|
||||||
|
directory: `+secretsRoot+`
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run", "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
warnings, ok := envelope["warnings"].([]any)
|
||||||
|
if !ok || len(warnings) != 1 {
|
||||||
|
t.Fatalf("warnings = %#v, want one warning", envelope["warnings"])
|
||||||
|
}
|
||||||
|
warning, ok := warnings[0].(map[string]any)
|
||||||
|
if !ok || !strings.Contains(fmt.Sprint(warning["message"]), name) {
|
||||||
|
t.Fatalf("warning = %#v, want secret name", warnings[0])
|
||||||
|
}
|
||||||
|
if strings.Contains(stdout.String(), "Warning:") || strings.Contains(stdout.String(), "process-value") || strings.Contains(stdout.String(), "secret-value") {
|
||||||
|
t.Fatalf("stdout exposed text warning or secret values: %q", stdout.String())
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunJSONFatalSetupErrorWritesNoJSON(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitError {
|
||||||
|
t.Fatalf("exit code = %d, want %d", code, exitError)
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "no such file or directory") {
|
||||||
|
t.Fatalf("stderr = %q, want setup error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunJSONPartialFailure(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
firstDestination := t.TempDir()
|
||||||
|
secondDestination := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write unmanaged file: %v", err)
|
||||||
|
}
|
||||||
|
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte(`
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive-one
|
||||||
|
backend: local
|
||||||
|
path: `+firstDestination+`
|
||||||
|
- id: archive-two
|
||||||
|
backend: local
|
||||||
|
path: `+secondDestination+`
|
||||||
|
`), 0o600); err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", configPath, "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitError {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitError, stderr.String())
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty for partial JSON result", stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
if envelope["command"] != "run" || envelope["ok"] != false {
|
||||||
|
t.Fatalf("envelope = %#v, want failed run envelope", envelope)
|
||||||
|
}
|
||||||
|
errors, ok := envelope["errors"].([]any)
|
||||||
|
if !ok || len(errors) != 1 {
|
||||||
|
t.Fatalf("errors = %#v, want one error", envelope["errors"])
|
||||||
|
}
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
summary, ok := result["summary"].(map[string]any)
|
||||||
|
if !ok || summary["status"] != "failed" || summary["failed"] != float64(1) {
|
||||||
|
t.Fatalf("summary = %#v, want failed summary", result["summary"])
|
||||||
|
}
|
||||||
|
actions, ok := result["actions"].([]any)
|
||||||
|
if !ok || len(actions) != 2 {
|
||||||
|
t.Fatalf("actions = %#v, want two actions", result["actions"])
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(secondDestination, storage.StateFileName)); err != nil {
|
||||||
|
t.Fatalf("second destination state stat error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
|
func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
@@ -249,3 +943,34 @@ func TestUnknownCommandIsUsageError(t *testing.T) {
|
|||||||
t.Fatalf("stderr = %q, want unknown command error", stderr.String())
|
t.Fatalf("stderr = %q, want unknown command error", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func manifestPaths(manifest producerbundle.Manifest) []string {
|
||||||
|
paths := make([]string, 0, len(manifest.Files))
|
||||||
|
for _, file := range manifest.Files {
|
||||||
|
paths = append(paths, file.Path)
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
|
func equalStrings(a, b []string) bool {
|
||||||
|
if len(a) != len(b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for index := range a {
|
||||||
|
if a[index] != b[index] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeCLIFile(t *testing.T, root, relative, body string) {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(root, filepath.FromSlash(relative))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -20,19 +20,28 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
|||||||
configPath := flags.String("config", "", "path to config file")
|
configPath := flags.String("config", "", "path to config file")
|
||||||
dryRun := flags.Bool("dry-run", false, "load and validate config without publishing")
|
dryRun := flags.Bool("dry-run", false, "load and validate config without publishing")
|
||||||
force := flags.Bool("force", false, "allow explicit destructive replacement for supported conflicts")
|
force := flags.Bool("force", false, "allow explicit destructive replacement for supported conflicts")
|
||||||
|
formatFlag := addFormatFlag(flags)
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
if rejectPositionalArgs(stderr, "run", flags.Args()) {
|
if rejectPositionalArgs(stderr, "run", flags.Args()) {
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
|
format, ok := parseOutputFormat(stderr, "run", *formatFlag)
|
||||||
|
if !ok {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
|
||||||
if err := app.Run(ctx, app.RunOptions{
|
if err := app.Run(ctx, app.RunOptions{
|
||||||
ConfigPath: *configPath,
|
ConfigPath: *configPath,
|
||||||
DryRun: *dryRun,
|
DryRun: *dryRun,
|
||||||
Force: *force,
|
Force: *force,
|
||||||
Stdout: stdout,
|
Stdout: stdout,
|
||||||
|
OutputFormat: format,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
|
if app.IsJSONOutput(format) && app.IsPartialResultError(err) {
|
||||||
|
return exitError
|
||||||
|
}
|
||||||
return fail(stderr, err)
|
return fail(stderr, err)
|
||||||
}
|
}
|
||||||
return exitOK
|
return exitOK
|
||||||
@@ -40,12 +49,14 @@ func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
|||||||
|
|
||||||
func printRunHelp(w io.Writer) {
|
func printRunHelp(w io.Writer) {
|
||||||
fmt.Fprint(w, `Usage:
|
fmt.Fprint(w, `Usage:
|
||||||
distributor run --config <path> [--dry-run] [--force]
|
distributor run --config <path> [--dry-run] [--force] [--format text|json]
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
--config <path> Path to config file
|
--config <path> Path to config file
|
||||||
--dry-run Load and validate config without publishing
|
--dry-run Load and validate config without publishing
|
||||||
--force Allow explicit destructive replacement for supported conflicts
|
--force Allow explicit destructive replacement for supported conflicts
|
||||||
|
--format text|json
|
||||||
|
Output format
|
||||||
|
|
||||||
Run discovers configured source bundles, plans each destination, publishes
|
Run discovers configured source bundles, plans each destination, publishes
|
||||||
selected outputs unless --dry-run is set, and prints a final status summary.
|
selected outputs unless --dry-run is set, and prints a final status summary.
|
||||||
|
|||||||
67
internal/cli/source_mode.go
Normal file
67
internal/cli/source_mode.go
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
type sourceDiagnosticArgs struct {
|
||||||
|
Path string
|
||||||
|
ConfigPath string
|
||||||
|
PipelineID string
|
||||||
|
BundlePath string
|
||||||
|
OutputFormat app.OutputFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSourceDiagnosticArgs(stderr io.Writer, command string, args []string) (sourceDiagnosticArgs, bool) {
|
||||||
|
flags := flag.NewFlagSet(command, flag.ContinueOnError)
|
||||||
|
flags.SetOutput(stderr)
|
||||||
|
configPath := flags.String("config", "", "path to config file")
|
||||||
|
pipelineID := flags.String("pipeline", "", "pipeline id")
|
||||||
|
bundlePath := flags.String("bundle", "", "source-root-relative bundle path")
|
||||||
|
formatFlag := addFormatFlag(flags)
|
||||||
|
if err := flags.Parse(args); err != nil {
|
||||||
|
return sourceDiagnosticArgs{}, false
|
||||||
|
}
|
||||||
|
format, ok := parseOutputFormat(stderr, command, *formatFlag)
|
||||||
|
if !ok {
|
||||||
|
return sourceDiagnosticArgs{}, false
|
||||||
|
}
|
||||||
|
path, ok := parseOptionalPathArg(stderr, command, flags.Args())
|
||||||
|
if !ok {
|
||||||
|
return sourceDiagnosticArgs{}, false
|
||||||
|
}
|
||||||
|
if !validateInspectModeOK(stderr, command, path, *configPath, *pipelineID, *bundlePath) {
|
||||||
|
return sourceDiagnosticArgs{}, false
|
||||||
|
}
|
||||||
|
return sourceDiagnosticArgs{
|
||||||
|
Path: path,
|
||||||
|
ConfigPath: *configPath,
|
||||||
|
PipelineID: *pipelineID,
|
||||||
|
BundlePath: *bundlePath,
|
||||||
|
OutputFormat: format,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateInspectModeOK(stderr io.Writer, command, path, configPath, pipelineID, bundlePath string) bool {
|
||||||
|
configMode := configPath != "" || pipelineID != "" || bundlePath != ""
|
||||||
|
if !configMode {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if path != "" {
|
||||||
|
fmt.Fprintf(stderr, "distributor: %s does not accept a local path with --config, --pipeline, or --bundle\n", command)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if configPath == "" {
|
||||||
|
fmt.Fprintf(stderr, "distributor: %s requires --config when --pipeline or --bundle is set\n", command)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if pipelineID == "" {
|
||||||
|
fmt.Fprintf(stderr, "distributor: %s requires --pipeline in config mode\n", command)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -13,11 +13,18 @@ func validateCommand(ctx context.Context, args []string, stdout, stderr io.Write
|
|||||||
printValidateHelp(stdout)
|
printValidateHelp(stdout)
|
||||||
return exitOK
|
return exitOK
|
||||||
}
|
}
|
||||||
path, ok := parseOptionalPathArg(stderr, "validate", args)
|
parsed, ok := parseSourceDiagnosticArgs(stderr, "validate", args)
|
||||||
if !ok {
|
if !ok {
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
if err := app.Validate(ctx, app.ValidateOptions{Path: path, Stdout: stdout}); err != nil {
|
if err := app.Validate(ctx, app.ValidateOptions{
|
||||||
|
Path: parsed.Path,
|
||||||
|
ConfigPath: parsed.ConfigPath,
|
||||||
|
PipelineID: parsed.PipelineID,
|
||||||
|
BundlePath: parsed.BundlePath,
|
||||||
|
Stdout: stdout,
|
||||||
|
OutputFormat: parsed.OutputFormat,
|
||||||
|
}); err != nil {
|
||||||
return fail(stderr, err)
|
return fail(stderr, err)
|
||||||
}
|
}
|
||||||
return exitOK
|
return exitOK
|
||||||
@@ -25,8 +32,16 @@ func validateCommand(ctx context.Context, args []string, stdout, stderr io.Write
|
|||||||
|
|
||||||
func printValidateHelp(w io.Writer) {
|
func printValidateHelp(w io.Writer) {
|
||||||
fmt.Fprint(w, `Usage:
|
fmt.Fprint(w, `Usage:
|
||||||
distributor validate <path>
|
distributor validate [--format text|json] <path>
|
||||||
|
distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
|
||||||
|
|
||||||
Validate a local source bundle directory or a tree containing source bundles.
|
Options:
|
||||||
|
--config <path> Path to config file for configured source validation
|
||||||
|
--pipeline <id> Pipeline id to validate in config mode
|
||||||
|
--bundle <path> Source-root-relative bundle path to validate
|
||||||
|
--format text|json Output format
|
||||||
|
|
||||||
|
Validate a local source bundle directory, a local source bundle tree, or a
|
||||||
|
configured pipeline source.
|
||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
@@ -13,16 +14,44 @@ func versionCommand(_ context.Context, args []string, stdout, stderr io.Writer)
|
|||||||
printVersionHelp(stdout)
|
printVersionHelp(stdout)
|
||||||
return exitOK
|
return exitOK
|
||||||
}
|
}
|
||||||
if rejectExtraArgs(stderr, "version", args) {
|
flags := flag.NewFlagSet("version", flag.ContinueOnError)
|
||||||
|
flags.SetOutput(stderr)
|
||||||
|
formatFlag := addFormatFlag(flags)
|
||||||
|
if err := flags.Parse(args); err != nil {
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
|
if rejectExtraArgs(stderr, "version", flags.Args()) {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
format, ok := parseOutputFormat(stderr, "version", *formatFlag)
|
||||||
|
if !ok {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if app.IsJSONOutput(format) {
|
||||||
|
err := app.WriteJSONEnvelope(stdout, "version", true, nil, versionResult{
|
||||||
|
Application: app.Name,
|
||||||
|
Version: app.Version,
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fail(stderr, err)
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
fmt.Fprintln(stdout, app.VersionString())
|
fmt.Fprintln(stdout, app.VersionString())
|
||||||
return exitOK
|
return exitOK
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type versionResult struct {
|
||||||
|
Application string `json:"application"`
|
||||||
|
Version string `json:"version"`
|
||||||
|
}
|
||||||
|
|
||||||
func printVersionHelp(w io.Writer) {
|
func printVersionHelp(w io.Writer) {
|
||||||
fmt.Fprint(w, `Usage:
|
fmt.Fprint(w, `Usage:
|
||||||
distributor version
|
distributor version [--format text|json]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--format text|json Output format
|
||||||
|
|
||||||
Print version information.
|
Print version information.
|
||||||
`)
|
`)
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ type Destination struct {
|
|||||||
SSH SSH `yaml:",inline"`
|
SSH SSH `yaml:",inline"`
|
||||||
Publish *PublishPolicy `yaml:"publish"`
|
Publish *PublishPolicy `yaml:"publish"`
|
||||||
Transform Transform `yaml:"transform"`
|
Transform Transform `yaml:"transform"`
|
||||||
|
PathMap PathMapping `yaml:"path_mapping"`
|
||||||
|
Links *Links `yaml:"links"`
|
||||||
Transfer TransferPolicy `yaml:"transfer"`
|
Transfer TransferPolicy `yaml:"transfer"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,6 +79,16 @@ type Transform struct {
|
|||||||
type MarkdownToHTML struct {
|
type MarkdownToHTML struct {
|
||||||
Enabled bool `yaml:"enabled"`
|
Enabled bool `yaml:"enabled"`
|
||||||
Mode string `yaml:"mode"`
|
Mode string `yaml:"mode"`
|
||||||
|
Input string `yaml:"input"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type PathMapping struct {
|
||||||
|
Mode string `yaml:"mode"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Links struct {
|
||||||
|
BaseURL string `yaml:"base_url"`
|
||||||
|
Primary string `yaml:"primary"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TransferPolicy struct {
|
type TransferPolicy struct {
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
|
|
||||||
const DefaultConfigPath = "/usr/local/etc/distributor/config.yml"
|
const DefaultConfigPath = "/usr/local/etc/distributor/config.yml"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -19,7 +21,19 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
TransformModeSidecar = "sidecar"
|
TransformModeSidecar = transform.MarkdownModeSidecar
|
||||||
|
TransformModeIndex = transform.MarkdownModeIndex
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
PathMappingPreserveRelative = "preserve_relative"
|
||||||
|
PathMappingFixed = "fixed"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
LinkPrimaryAuto = "auto"
|
||||||
|
LinkPrimaryHTML = "html"
|
||||||
|
LinkPrimarySource = "source"
|
||||||
)
|
)
|
||||||
|
|
||||||
const DefaultS3Region = "us-east-1"
|
const DefaultS3Region = "us-east-1"
|
||||||
@@ -37,6 +51,15 @@ func ApplyDefaults(cfg *Config) {
|
|||||||
if destination.Publish == nil {
|
if destination.Publish == nil {
|
||||||
destination.Publish = &PublishPolicy{Source: true}
|
destination.Publish = &PublishPolicy{Source: true}
|
||||||
}
|
}
|
||||||
|
if destination.Transform.MarkdownToHTML != nil && destination.Transform.MarkdownToHTML.Mode == "" {
|
||||||
|
destination.Transform.MarkdownToHTML.Mode = TransformModeSidecar
|
||||||
|
}
|
||||||
|
if destination.PathMap.Mode == "" {
|
||||||
|
destination.PathMap.Mode = PathMappingPreserveRelative
|
||||||
|
}
|
||||||
|
if destination.Links != nil && destination.Links.Primary == "" {
|
||||||
|
destination.Links.Primary = LinkPrimaryAuto
|
||||||
|
}
|
||||||
if destination.Transfer.OnDestinationSame == "" {
|
if destination.Transfer.OnDestinationSame == "" {
|
||||||
destination.Transfer.OnDestinationSame = TransferActionSkip
|
destination.Transfer.OnDestinationSame = TransferActionSkip
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -94,6 +94,120 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadFileValidMarkdownIndexConfig(t *testing.T) {
|
||||||
|
cfg := loadConfig(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: static-site
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/reports
|
||||||
|
destinations:
|
||||||
|
- id: web
|
||||||
|
backend: local
|
||||||
|
path: /srv/www/reports
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: index
|
||||||
|
input: report.md
|
||||||
|
`)
|
||||||
|
|
||||||
|
markdown := cfg.Pipelines[0].Destinations[0].Transform.MarkdownToHTML
|
||||||
|
if markdown == nil || markdown.Mode != TransformModeIndex || markdown.Input != "report.md" {
|
||||||
|
t.Fatalf("markdown config = %#v, want index input", markdown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileDefaultsMarkdownModeToSidecar(t *testing.T) {
|
||||||
|
cfg := loadConfig(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: static-site
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/reports
|
||||||
|
destinations:
|
||||||
|
- id: web
|
||||||
|
backend: local
|
||||||
|
path: /srv/www/reports
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
`)
|
||||||
|
|
||||||
|
markdown := cfg.Pipelines[0].Destinations[0].Transform.MarkdownToHTML
|
||||||
|
if markdown == nil || markdown.Mode != TransformModeSidecar {
|
||||||
|
t.Fatalf("markdown mode = %#v, want sidecar default", markdown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileDefaultsPathMappingToPreserveRelative(t *testing.T) {
|
||||||
|
cfg := loadConfig(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /destination
|
||||||
|
`)
|
||||||
|
|
||||||
|
if got, want := cfg.Pipelines[0].Destinations[0].PathMap.Mode, PathMappingPreserveRelative; got != want {
|
||||||
|
t.Fatalf("path mapping mode = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileAcceptsFixedPathMapping(t *testing.T) {
|
||||||
|
cfg := loadConfig(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: latest
|
||||||
|
backend: local
|
||||||
|
path: /destination/latest
|
||||||
|
path_mapping:
|
||||||
|
mode: fixed
|
||||||
|
`)
|
||||||
|
|
||||||
|
if got, want := cfg.Pipelines[0].Destinations[0].PathMap.Mode, PathMappingFixed; got != want {
|
||||||
|
t.Fatalf("path mapping mode = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileDefaultsLinksPrimaryToAuto(t *testing.T) {
|
||||||
|
cfg := loadConfig(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: web
|
||||||
|
backend: local
|
||||||
|
path: /destination
|
||||||
|
links:
|
||||||
|
base_url: https://reports.example.com/archive
|
||||||
|
`)
|
||||||
|
|
||||||
|
links := cfg.Pipelines[0].Destinations[0].Links
|
||||||
|
if links == nil {
|
||||||
|
t.Fatal("links = nil, want config")
|
||||||
|
}
|
||||||
|
if links.BaseURL != "https://reports.example.com/archive" || links.Primary != LinkPrimaryAuto {
|
||||||
|
t.Fatalf("links = %#v, want base URL with auto primary", links)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadFileValidBackendConfigs(t *testing.T) {
|
func TestLoadFileValidBackendConfigs(t *testing.T) {
|
||||||
tests := map[string]string{
|
tests := map[string]string{
|
||||||
"local": `
|
"local": `
|
||||||
@@ -464,7 +578,9 @@ func TestExampleConfigsLoad(t *testing.T) {
|
|||||||
"../../examples/local-to-local.yml",
|
"../../examples/local-to-local.yml",
|
||||||
"../../examples/local-publish.yml",
|
"../../examples/local-publish.yml",
|
||||||
"../../examples/local-html.yml",
|
"../../examples/local-html.yml",
|
||||||
|
"../../examples/local-index.yml",
|
||||||
"../../examples/fan-out.yml",
|
"../../examples/fan-out.yml",
|
||||||
|
"../../examples/archive-and-latest.yml",
|
||||||
"../../examples/ssh-destination.yml",
|
"../../examples/ssh-destination.yml",
|
||||||
"../../examples/s3-destination.yml",
|
"../../examples/s3-destination.yml",
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
var idPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`)
|
var idPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`)
|
||||||
@@ -58,6 +60,8 @@ func Validate(cfg Config) error {
|
|||||||
|
|
||||||
errs = validateDestinationBackend(errs, destinationContext, destination)
|
errs = validateDestinationBackend(errs, destinationContext, destination)
|
||||||
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
|
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
|
||||||
|
errs = validatePathMapping(errs, destinationContext+".path_mapping", destination.PathMap)
|
||||||
|
errs = validateLinks(errs, destinationContext+".links", destination.Links)
|
||||||
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
|
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -149,21 +153,52 @@ func ValidatePublishTransformPolicy(publish PublishPolicy, transform Transform)
|
|||||||
if transform.MarkdownToHTML == nil {
|
if transform.MarkdownToHTML == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
mode := transform.MarkdownToHTML.Mode
|
||||||
|
if mode == "" {
|
||||||
|
mode = TransformModeSidecar
|
||||||
|
}
|
||||||
|
if mode != TransformModeSidecar && mode != TransformModeIndex {
|
||||||
|
return fmt.Errorf("transform.markdown_to_html.mode must be %s or %s", TransformModeSidecar, TransformModeIndex)
|
||||||
|
}
|
||||||
|
if transform.MarkdownToHTML.Input != "" && !transform.MarkdownToHTML.Enabled {
|
||||||
|
return fmt.Errorf("transform.markdown_to_html.input requires transform.markdown_to_html.enabled to be true")
|
||||||
|
}
|
||||||
|
if transform.MarkdownToHTML.Input != "" && mode != TransformModeIndex {
|
||||||
|
return fmt.Errorf("transform.markdown_to_html.input is only valid when mode is %s", TransformModeIndex)
|
||||||
|
}
|
||||||
|
if transform.MarkdownToHTML.Enabled && !publish.HTML {
|
||||||
|
return fmt.Errorf("transform.markdown_to_html.enabled requires publish.html to be true")
|
||||||
|
}
|
||||||
if publish.HTML && !transform.MarkdownToHTML.Enabled {
|
if publish.HTML && !transform.MarkdownToHTML.Enabled {
|
||||||
return fmt.Errorf("transform.markdown_to_html.enabled must be true when publish.html is true")
|
return fmt.Errorf("transform.markdown_to_html.enabled must be true when publish.html is true")
|
||||||
}
|
}
|
||||||
if publish.HTML && transform.MarkdownToHTML.Mode != TransformModeSidecar {
|
|
||||||
return fmt.Errorf("transform.markdown_to_html.mode must be %s", TransformModeSidecar)
|
|
||||||
}
|
|
||||||
if transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != TransformModeSidecar {
|
|
||||||
return fmt.Errorf("transform.markdown_to_html.mode must be %s", TransformModeSidecar)
|
|
||||||
}
|
|
||||||
if !transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != "" && transform.MarkdownToHTML.Mode != TransformModeSidecar {
|
|
||||||
return fmt.Errorf("transform.markdown_to_html.mode must be %s", TransformModeSidecar)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validatePathMapping(errs ValidationErrors, context string, mapping PathMapping) ValidationErrors {
|
||||||
|
if mapping.Mode != PathMappingPreserveRelative && mapping.Mode != PathMappingFixed {
|
||||||
|
errs = append(errs, context+".mode must be "+PathMappingPreserveRelative+" or "+PathMappingFixed)
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateLinks(errs ValidationErrors, context string, links *Links) ValidationErrors {
|
||||||
|
if links == nil {
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
if links.BaseURL == "" {
|
||||||
|
errs = append(errs, context+".base_url is required")
|
||||||
|
} else if err := link.ValidateHTTPURL(links.BaseURL); err != nil {
|
||||||
|
errs = append(errs, context+".base_url "+err.Error())
|
||||||
|
}
|
||||||
|
switch links.Primary {
|
||||||
|
case LinkPrimaryAuto, LinkPrimaryHTML, LinkPrimarySource:
|
||||||
|
default:
|
||||||
|
errs = append(errs, context+".primary must be "+LinkPrimaryAuto+", "+LinkPrimaryHTML+", or "+LinkPrimarySource)
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
|
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
|
||||||
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
|
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
|
||||||
errs = append(errs, context+".on_destination_same must be skip or fail")
|
errs = append(errs, context+".on_destination_same must be skip or fail")
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
func TestValidatePublishTransformPolicy(t *testing.T) {
|
func TestValidatePublishTransformPolicy(t *testing.T) {
|
||||||
tests := publishTransformPolicyCases()
|
tests := publishTransformPolicyCases()
|
||||||
@@ -70,6 +73,103 @@ func TestValidateAcceptsForceReplacementTransferActions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidatePathMapping(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mode string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "preserve relative", mode: PathMappingPreserveRelative},
|
||||||
|
{name: "fixed", mode: PathMappingFixed},
|
||||||
|
{name: "invalid", mode: "archive", wantErr: true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
cfg := Config{Pipelines: []Pipeline{{
|
||||||
|
ID: "reports",
|
||||||
|
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||||
|
Destinations: []Destination{{
|
||||||
|
ID: "archive",
|
||||||
|
Backend: BackendLocal,
|
||||||
|
Path: "/destination",
|
||||||
|
PathMap: PathMapping{Mode: tt.mode},
|
||||||
|
}},
|
||||||
|
}}}
|
||||||
|
ApplyDefaults(&cfg)
|
||||||
|
err := Validate(cfg)
|
||||||
|
if tt.wantErr && err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !tt.wantErr && err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateLinks(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
links *Links
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "absent links"},
|
||||||
|
{name: "http", links: &Links{BaseURL: "http://reports.example.com/archive", Primary: LinkPrimaryAuto}},
|
||||||
|
{name: "https", links: &Links{BaseURL: "https://reports.example.com/archive/", Primary: LinkPrimaryHTML}},
|
||||||
|
{name: "source primary", links: &Links{BaseURL: "https://reports.example.com", Primary: LinkPrimarySource}},
|
||||||
|
{name: "missing base", links: &Links{Primary: LinkPrimaryAuto}, wantErr: true},
|
||||||
|
{name: "ftp scheme", links: &Links{BaseURL: "ftp://reports.example.com", Primary: LinkPrimaryAuto}, wantErr: true},
|
||||||
|
{name: "missing host", links: &Links{BaseURL: "https:///archive", Primary: LinkPrimaryAuto}, wantErr: true},
|
||||||
|
{name: "query", links: &Links{BaseURL: "https://reports.example.com/archive?preview=1", Primary: LinkPrimaryAuto}, wantErr: true},
|
||||||
|
{name: "fragment", links: &Links{BaseURL: "https://reports.example.com/archive#top", Primary: LinkPrimaryAuto}, wantErr: true},
|
||||||
|
{name: "invalid primary", links: &Links{BaseURL: "https://reports.example.com", Primary: "document"}, wantErr: true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
cfg := Config{Pipelines: []Pipeline{{
|
||||||
|
ID: "reports",
|
||||||
|
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||||
|
Destinations: []Destination{{
|
||||||
|
ID: "web",
|
||||||
|
Backend: BackendLocal,
|
||||||
|
Path: "/destination",
|
||||||
|
Links: tt.links,
|
||||||
|
}},
|
||||||
|
}}}
|
||||||
|
ApplyDefaults(&cfg)
|
||||||
|
err := Validate(cfg)
|
||||||
|
if tt.wantErr && err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !tt.wantErr && err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateLinksReportsFieldContext(t *testing.T) {
|
||||||
|
cfg := Config{Pipelines: []Pipeline{{
|
||||||
|
ID: "reports",
|
||||||
|
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||||
|
Destinations: []Destination{{
|
||||||
|
ID: "web",
|
||||||
|
Backend: BackendLocal,
|
||||||
|
Path: "/destination",
|
||||||
|
Links: &Links{BaseURL: "https://reports.example.com/archive?preview=1", Primary: LinkPrimaryAuto},
|
||||||
|
}},
|
||||||
|
}}}
|
||||||
|
ApplyDefaults(&cfg)
|
||||||
|
err := Validate(cfg)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want error")
|
||||||
|
}
|
||||||
|
want := "pipelines[0].destinations[0].links.base_url must not include a query string"
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("Validate() error = %q, want %q", err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type publishTransformPolicyCase struct {
|
type publishTransformPolicyCase struct {
|
||||||
name string
|
name string
|
||||||
publish PublishPolicy
|
publish PublishPolicy
|
||||||
@@ -91,6 +191,30 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
|
|||||||
Mode: TransformModeSidecar,
|
Mode: TransformModeSidecar,
|
||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "html only default mode allowed",
|
||||||
|
publish: PublishPolicy{HTML: true},
|
||||||
|
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "html only index allowed",
|
||||||
|
publish: PublishPolicy{HTML: true},
|
||||||
|
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
Mode: TransformModeIndex,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "html only index input allowed",
|
||||||
|
publish: PublishPolicy{HTML: true},
|
||||||
|
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
Mode: TransformModeIndex,
|
||||||
|
Input: "report.md",
|
||||||
|
}},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "source and html sidecar allowed",
|
name: "source and html sidecar allowed",
|
||||||
publish: PublishPolicy{Source: true, HTML: true},
|
publish: PublishPolicy{Source: true, HTML: true},
|
||||||
@@ -127,6 +251,15 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
|
|||||||
}},
|
}},
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "source only enabled transform rejected",
|
||||||
|
publish: PublishPolicy{Source: true},
|
||||||
|
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
Mode: TransformModeSidecar,
|
||||||
|
}},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "enabled markdown wrong mode rejected",
|
name: "enabled markdown wrong mode rejected",
|
||||||
publish: PublishPolicy{Source: true},
|
publish: PublishPolicy{Source: true},
|
||||||
@@ -136,6 +269,16 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
|
|||||||
}},
|
}},
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "sidecar input rejected",
|
||||||
|
publish: PublishPolicy{HTML: true},
|
||||||
|
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
Mode: TransformModeSidecar,
|
||||||
|
Input: "report.md",
|
||||||
|
}},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "disabled markdown empty mode allowed",
|
name: "disabled markdown empty mode allowed",
|
||||||
publish: PublishPolicy{Source: true},
|
publish: PublishPolicy{Source: true},
|
||||||
@@ -151,6 +294,24 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
|
|||||||
Mode: TransformModeSidecar,
|
Mode: TransformModeSidecar,
|
||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "disabled markdown index mode allowed",
|
||||||
|
publish: PublishPolicy{Source: true},
|
||||||
|
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||||
|
Enabled: false,
|
||||||
|
Mode: TransformModeIndex,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "disabled markdown input rejected",
|
||||||
|
publish: PublishPolicy{Source: true},
|
||||||
|
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||||
|
Enabled: false,
|
||||||
|
Mode: TransformModeIndex,
|
||||||
|
Input: "report.md",
|
||||||
|
}},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "disabled markdown wrong mode rejected",
|
name: "disabled markdown wrong mode rejected",
|
||||||
publish: PublishPolicy{Source: true},
|
publish: PublishPolicy{Source: true},
|
||||||
|
|||||||
26
internal/link/url.go
Normal file
26
internal/link/url.go
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ValidateHTTPURL(value string) error {
|
||||||
|
parsed, err := url.Parse(value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("must be a valid URL")
|
||||||
|
}
|
||||||
|
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||||
|
return fmt.Errorf("must use http or https")
|
||||||
|
}
|
||||||
|
if parsed.Host == "" {
|
||||||
|
return fmt.Errorf("must include a host")
|
||||||
|
}
|
||||||
|
if parsed.RawQuery != "" {
|
||||||
|
return fmt.Errorf("must not include a query string")
|
||||||
|
}
|
||||||
|
if parsed.Fragment != "" {
|
||||||
|
return fmt.Errorf("must not include a fragment")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
39
internal/link/url_test.go
Normal file
39
internal/link/url_test.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateHTTPURL(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "http", value: "http://reports.example.com/archive"},
|
||||||
|
{name: "https", value: "https://reports.example.com/archive"},
|
||||||
|
{name: "missing host", value: "https:///archive", wantErr: "must include a host"},
|
||||||
|
{name: "unsupported scheme", value: "ftp://reports.example.com/archive", wantErr: "must use http or https"},
|
||||||
|
{name: "query string", value: "https://reports.example.com/archive?preview=1", wantErr: "must not include a query string"},
|
||||||
|
{name: "fragment", value: "https://reports.example.com/archive#top", wantErr: "must not include a fragment"},
|
||||||
|
{name: "parse failure", value: "http://[::1", wantErr: "must be a valid URL"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := ValidateHTTPURL(tt.value)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ValidateHTTPURL() error = %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ValidateHTTPURL() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("ValidateHTTPURL() error = %q, want %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -23,7 +23,7 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
|||||||
if plan.ExistingState == nil {
|
if plan.ExistingState == nil {
|
||||||
return fmt.Errorf("replace requires existing destination state")
|
return fmt.Errorf("replace requires existing destination state")
|
||||||
}
|
}
|
||||||
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, existingManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, stateOutputManagedPaths(plan.ExistingState.Outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||||
@@ -41,7 +41,7 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
|||||||
|
|
||||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||||
cleanup := func() {
|
cleanup := func() {
|
||||||
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, managedOutputPaths(writtenOutputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(writtenOutputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||||
}
|
}
|
||||||
for _, output := range plan.Outputs {
|
for _, output := range plan.Outputs {
|
||||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
||||||
@@ -76,7 +76,10 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
|||||||
DestinationID: req.DestinationID,
|
DestinationID: req.DestinationID,
|
||||||
PublishedAt: time.Now().UTC(),
|
PublishedAt: time.Now().UTC(),
|
||||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
||||||
Outputs: stateOutputs(plan.Outputs),
|
Outputs: StateOutputFiles(plan.Outputs),
|
||||||
|
}
|
||||||
|
if plan.PrimaryURL != "" {
|
||||||
|
destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
||||||
}
|
}
|
||||||
if err := state.Validate(destinationState); err != nil {
|
if err := state.Validate(destinationState); err != nil {
|
||||||
cleanup()
|
cleanup()
|
||||||
@@ -99,11 +102,3 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func existingManagedOutputPaths(destinationState state.DistributorState) []string {
|
|
||||||
paths := make([]string, 0, len(destinationState.Outputs))
|
|
||||||
for _, output := range destinationState.Outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,13 +2,11 @@ package publish
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||||
)
|
)
|
||||||
@@ -25,7 +23,7 @@ func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
|||||||
name: "unmanaged content",
|
name: "unmanaged content",
|
||||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
writeFakeFile(t, backend, "bundle/old.txt", "old")
|
testutil.WriteFakeFile(t, backend, "bundle/old.txt", "old")
|
||||||
},
|
},
|
||||||
transfer: defaultTransfer(),
|
transfer: defaultTransfer(),
|
||||||
wantReason: "fail_unmanaged",
|
wantReason: "fail_unmanaged",
|
||||||
@@ -37,7 +35,7 @@ func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
conflict := source
|
conflict := source
|
||||||
conflict.ID = "other.source"
|
conflict.ID = "other.source"
|
||||||
writeFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||||
},
|
},
|
||||||
transfer: conflictReplaceTransfer(),
|
transfer: conflictReplaceTransfer(),
|
||||||
wantReason: "requires --force",
|
wantReason: "requires --force",
|
||||||
@@ -48,7 +46,7 @@ func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
|||||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
conflict := testutil.ValidManifest(testutil.BundleOptions{Files: []testutil.SourceFile{{Path: "report.md", Data: "# Different\n"}}})
|
conflict := testutil.ValidManifest(testutil.BundleOptions{Files: []testutil.SourceFile{{Path: "report.md", Data: "# Different\n"}}})
|
||||||
writeFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||||
},
|
},
|
||||||
transfer: conflictReplaceTransfer(),
|
transfer: conflictReplaceTransfer(),
|
||||||
wantReason: "requires --force",
|
wantReason: "requires --force",
|
||||||
@@ -58,7 +56,7 @@ func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
|||||||
name: "pipeline mismatch",
|
name: "pipeline mismatch",
|
||||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
writeFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{PipelineID: "other-pipeline"})
|
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{PipelineID: "other-pipeline"})
|
||||||
},
|
},
|
||||||
transfer: conflictReplaceTransfer(),
|
transfer: conflictReplaceTransfer(),
|
||||||
wantReason: "requires --force",
|
wantReason: "requires --force",
|
||||||
@@ -68,7 +66,7 @@ func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
|||||||
name: "destination mismatch",
|
name: "destination mismatch",
|
||||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
writeFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{DestinationID: "other-destination"})
|
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{DestinationID: "other-destination"})
|
||||||
},
|
},
|
||||||
transfer: conflictReplaceTransfer(),
|
transfer: conflictReplaceTransfer(),
|
||||||
wantReason: "requires --force",
|
wantReason: "requires --force",
|
||||||
@@ -80,7 +78,7 @@ func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
newer := source
|
newer := source
|
||||||
newer.Created = newer.Created.AddDate(0, 0, 1)
|
newer.Created = newer.Created.AddDate(0, 0, 1)
|
||||||
writeFakeDestinationState(t, backend, "bundle", newer, testutil.DestinationStateOptions{})
|
testutil.WriteFakeDestinationState(t, backend, "bundle", newer, testutil.DestinationStateOptions{})
|
||||||
},
|
},
|
||||||
transfer: newerReplaceTransfer(),
|
transfer: newerReplaceTransfer(),
|
||||||
wantReason: "requires --force",
|
wantReason: "requires --force",
|
||||||
@@ -121,7 +119,7 @@ func TestBuildRequiresConflictPolicyForStateConflicts(t *testing.T) {
|
|||||||
destinationBackend := fake.New()
|
destinationBackend := fake.New()
|
||||||
conflict := sourceBundle.Manifest
|
conflict := sourceBundle.Manifest
|
||||||
conflict.ID = "other.source"
|
conflict.ID = "other.source"
|
||||||
writeFakeDestinationState(t, destinationBackend, "bundle", conflict, testutil.DestinationStateOptions{})
|
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||||
|
|
||||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
||||||
req.Force = true
|
req.Force = true
|
||||||
@@ -135,10 +133,10 @@ func TestExecuteForcedReplacementDeletesOnlyBundlePath(t *testing.T) {
|
|||||||
sourceBackend := fake.New()
|
sourceBackend := fake.New()
|
||||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||||
destinationBackend := fake.New()
|
destinationBackend := fake.New()
|
||||||
writeFakeFile(t, destinationBackend, "bundle/old.txt", "old")
|
testutil.WriteFakeFile(t, destinationBackend, "bundle/old.txt", "old")
|
||||||
writeFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
|
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
|
||||||
writeFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||||
writeFakeFile(t, destinationBackend, "outside.txt", "outside")
|
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "outside")
|
||||||
|
|
||||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
||||||
req.Force = true
|
req.Force = true
|
||||||
@@ -152,11 +150,11 @@ func TestExecuteForcedReplacementDeletesOnlyBundlePath(t *testing.T) {
|
|||||||
if err := Execute(context.Background(), req, plan); err != nil {
|
if err := Execute(context.Background(), req, plan); err != nil {
|
||||||
t.Fatalf("Execute() error = %v", err)
|
t.Fatalf("Execute() error = %v", err)
|
||||||
}
|
}
|
||||||
assertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
||||||
assertFakeMissing(t, destinationBackend, "bundle/old.txt")
|
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.txt")
|
||||||
assertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
|
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
|
||||||
assertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||||
assertFakeFile(t, destinationBackend, "outside.txt", "outside")
|
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "outside")
|
||||||
}
|
}
|
||||||
|
|
||||||
func forceRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, transfer config.TransferPolicy) Request {
|
func forceRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, transfer config.TransferPolicy) Request {
|
||||||
@@ -193,49 +191,3 @@ func newerReplaceTransfer() config.TransferPolicy {
|
|||||||
transfer.OnDestinationNewer = config.TransferActionReplace
|
transfer.OnDestinationNewer = config.TransferActionReplace
|
||||||
return transfer
|
return transfer
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeFakeDestinationState(t *testing.T, backend *fake.Backend, relative string, manifest bundle.Manifest, opts testutil.DestinationStateOptions) {
|
|
||||||
t.Helper()
|
|
||||||
destinationState := testutil.DestinationState(manifest, opts)
|
|
||||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal destination state: %v", err)
|
|
||||||
}
|
|
||||||
statePath, err := storage.StatePath(relative)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("state path: %v", err)
|
|
||||||
}
|
|
||||||
writeFakeFile(t, backend, statePath, string(append(data, '\n')))
|
|
||||||
for _, output := range destinationState.Outputs {
|
|
||||||
path, err := storage.Join(relative, output.Path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("join output path: %v", err)
|
|
||||||
}
|
|
||||||
writeFakeFile(t, backend, path, "old")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeFakeFile(t *testing.T, backend *fake.Backend, path, data string) {
|
|
||||||
t.Helper()
|
|
||||||
if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
|
|
||||||
t.Fatalf("write fake file %s: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertFakeFile(t *testing.T, backend *fake.Backend, path, want string) {
|
|
||||||
t.Helper()
|
|
||||||
data, err := backend.ReadFile(context.Background(), path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read fake file %s: %v", path, err)
|
|
||||||
}
|
|
||||||
if got := string(data); got != want {
|
|
||||||
t.Fatalf("fake file %s = %q, want %q", path, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertFakeMissing(t *testing.T, backend *fake.Backend, path string) {
|
|
||||||
t.Helper()
|
|
||||||
if _, err := backend.Stat(context.Background(), path); !storage.IsNotFound(err) {
|
|
||||||
t.Fatalf("fake file %s stat error = %v, want not found", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
120
internal/publish/links.go
Normal file
120
internal/publish/links.go
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
|
)
|
||||||
|
|
||||||
|
func PlanLinks(req Request, outputs []Output) ([]Output, string, error) {
|
||||||
|
if req.Links == nil {
|
||||||
|
return outputs, "", nil
|
||||||
|
}
|
||||||
|
if err := link.ValidateHTTPURL(req.Links.BaseURL); err != nil {
|
||||||
|
return nil, "", fmt.Errorf("link base URL: %w", err)
|
||||||
|
}
|
||||||
|
linked := make([]Output, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
outputURL, err := OutputURL(req.Links.BaseURL, req.DestinationBundlePath, output.DestinationPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
output.URL = outputURL
|
||||||
|
linked = append(linked, output)
|
||||||
|
}
|
||||||
|
return linked, primaryURL(linked, req.Links.Primary), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func OutputURL(baseURL, destinationBundlePath, outputPath string) (string, error) {
|
||||||
|
parsed, err := url.Parse(baseURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("link base URL: %w", err)
|
||||||
|
}
|
||||||
|
segments := pathSegments(parsed.Path)
|
||||||
|
segments = append(segments, pathSegments(destinationBundlePath)...)
|
||||||
|
outputSegments := pathSegments(outputPath)
|
||||||
|
trailingSlash := false
|
||||||
|
if len(outputSegments) > 0 && outputSegments[len(outputSegments)-1] == "index.html" {
|
||||||
|
outputSegments = outputSegments[:len(outputSegments)-1]
|
||||||
|
trailingSlash = true
|
||||||
|
}
|
||||||
|
segments = append(segments, outputSegments...)
|
||||||
|
parsed.Path = urlPath(segments, trailingSlash)
|
||||||
|
parsed.RawPath = ""
|
||||||
|
return parsed.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathSegments(value string) []string {
|
||||||
|
trimmed := strings.Trim(value, "/")
|
||||||
|
if trimmed == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return strings.Split(trimmed, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func urlPath(segments []string, trailingSlash bool) string {
|
||||||
|
if len(segments) == 0 {
|
||||||
|
return "/"
|
||||||
|
}
|
||||||
|
path := "/" + strings.Join(segments, "/")
|
||||||
|
if trailingSlash && !strings.HasSuffix(path, "/") {
|
||||||
|
path += "/"
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func primaryURL(outputs []Output, policy string) string {
|
||||||
|
switch policy {
|
||||||
|
case config.LinkPrimaryHTML:
|
||||||
|
return firstGeneratedHTMLURL(outputs)
|
||||||
|
case config.LinkPrimarySource:
|
||||||
|
return firstSourceURL(outputs)
|
||||||
|
default:
|
||||||
|
if url := firstIndexURL(outputs); url != "" {
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
if url := firstGeneratedHTMLURL(outputs); url != "" {
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
return firstSourceURL(outputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstIndexURL(outputs []Output) string {
|
||||||
|
for _, output := range outputs {
|
||||||
|
if isIndexOutput(output.DestinationPath) {
|
||||||
|
return output.URL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstGeneratedHTMLURL(outputs []Output) string {
|
||||||
|
for _, output := range outputs {
|
||||||
|
if output.Kind == state.OutputKindGenerated && isHTMLOutput(output.DestinationPath) {
|
||||||
|
return output.URL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstSourceURL(outputs []Output) string {
|
||||||
|
for _, output := range outputs {
|
||||||
|
if output.Kind == state.OutputKindSource {
|
||||||
|
return output.URL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func isIndexOutput(path string) bool {
|
||||||
|
return path == "index.html" || strings.HasSuffix(path, "/index.html")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isHTMLOutput(path string) bool {
|
||||||
|
return strings.HasSuffix(path, ".html")
|
||||||
|
}
|
||||||
169
internal/publish/links_test.go
Normal file
169
internal/publish/links_test.go
Normal file
@@ -0,0 +1,169 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOutputURLUsesURLPathSemantics(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
baseURL string
|
||||||
|
destinationBundlePath string
|
||||||
|
outputPath string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "nested non index",
|
||||||
|
baseURL: "https://reports.example.com/archive",
|
||||||
|
destinationBundlePath: "daily/brentwood",
|
||||||
|
outputPath: "report.html",
|
||||||
|
want: "https://reports.example.com/archive/daily/brentwood/report.html",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested index",
|
||||||
|
baseURL: "https://reports.example.com/archive",
|
||||||
|
destinationBundlePath: "daily/brentwood",
|
||||||
|
outputPath: "index.html",
|
||||||
|
want: "https://reports.example.com/archive/daily/brentwood/",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fixed index",
|
||||||
|
baseURL: "https://reports.example.com/latest",
|
||||||
|
destinationBundlePath: "",
|
||||||
|
outputPath: "index.html",
|
||||||
|
want: "https://reports.example.com/latest/",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "escaped segments",
|
||||||
|
baseURL: "https://reports.example.com/archive root",
|
||||||
|
destinationBundlePath: "daily reports",
|
||||||
|
outputPath: "morning report.html",
|
||||||
|
want: "https://reports.example.com/archive%20root/daily%20reports/morning%20report.html",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested output index",
|
||||||
|
baseURL: "https://reports.example.com",
|
||||||
|
destinationBundlePath: "daily",
|
||||||
|
outputPath: "site/index.html",
|
||||||
|
want: "https://reports.example.com/daily/site/",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := OutputURL(tt.baseURL, tt.destinationBundlePath, tt.outputPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OutputURL() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("OutputURL() = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanLinksSelectsPrimaryURL(t *testing.T) {
|
||||||
|
outputs := []Output{
|
||||||
|
{
|
||||||
|
DestinationPath: "report.md",
|
||||||
|
Kind: state.OutputKindSource,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
DestinationPath: "report.html",
|
||||||
|
Kind: state.OutputKindGenerated,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
DestinationPath: "index.html",
|
||||||
|
Kind: state.OutputKindGenerated,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
primary string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "auto prefers index", primary: config.LinkPrimaryAuto, want: "https://reports.example.com/daily/"},
|
||||||
|
{name: "html uses first generated html", primary: config.LinkPrimaryHTML, want: "https://reports.example.com/daily/report.html"},
|
||||||
|
{name: "source uses first source", primary: config.LinkPrimarySource, want: "https://reports.example.com/daily/report.md"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
linked, primaryURL, err := PlanLinks(Request{
|
||||||
|
DestinationBundlePath: "daily",
|
||||||
|
Links: &config.Links{
|
||||||
|
BaseURL: "https://reports.example.com",
|
||||||
|
Primary: tt.primary,
|
||||||
|
},
|
||||||
|
}, outputs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanLinks() error = %v", err)
|
||||||
|
}
|
||||||
|
if primaryURL != tt.want {
|
||||||
|
t.Fatalf("primary URL = %q, want %q", primaryURL, tt.want)
|
||||||
|
}
|
||||||
|
for index, output := range linked {
|
||||||
|
if output.URL == "" {
|
||||||
|
t.Fatalf("linked output %d has empty URL", index)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanLinksReturnsNoPrimaryWhenPolicyHasNoMatch(t *testing.T) {
|
||||||
|
linked, primaryURL, err := PlanLinks(Request{
|
||||||
|
DestinationBundlePath: "daily",
|
||||||
|
Links: &config.Links{
|
||||||
|
BaseURL: "https://reports.example.com",
|
||||||
|
Primary: config.LinkPrimaryHTML,
|
||||||
|
},
|
||||||
|
}, []Output{{
|
||||||
|
DestinationPath: "report.md",
|
||||||
|
Kind: state.OutputKindSource,
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanLinks() error = %v", err)
|
||||||
|
}
|
||||||
|
if primaryURL != "" {
|
||||||
|
t.Fatalf("primary URL = %q, want empty", primaryURL)
|
||||||
|
}
|
||||||
|
if linked[0].URL != "https://reports.example.com/daily/report.md" {
|
||||||
|
t.Fatalf("linked URL = %q", linked[0].URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanLinksLeavesOutputsUnchangedWithoutConfig(t *testing.T) {
|
||||||
|
outputs := []Output{{DestinationPath: "report.md", Kind: state.OutputKindSource}}
|
||||||
|
linked, primaryURL, err := PlanLinks(Request{}, outputs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanLinks() error = %v", err)
|
||||||
|
}
|
||||||
|
if primaryURL != "" {
|
||||||
|
t.Fatalf("primary URL = %q, want empty", primaryURL)
|
||||||
|
}
|
||||||
|
if linked[0].URL != "" {
|
||||||
|
t.Fatalf("output URL = %q, want empty", linked[0].URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanLinksValidatesBaseURLBeforePlanning(t *testing.T) {
|
||||||
|
_, _, err := PlanLinks(Request{
|
||||||
|
DestinationBundlePath: "daily",
|
||||||
|
Links: &config.Links{
|
||||||
|
BaseURL: "https://reports.example.com/archive?preview=1",
|
||||||
|
Primary: config.LinkPrimaryAuto,
|
||||||
|
},
|
||||||
|
}, []Output{{
|
||||||
|
DestinationPath: "report.md",
|
||||||
|
Kind: state.OutputKindSource,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("PlanLinks() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "link base URL: must not include a query string") {
|
||||||
|
t.Fatalf("PlanLinks() error = %q, want link base URL context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,6 +26,10 @@ func PlanOutputs(ctx context.Context, req Request) ([]Output, error) {
|
|||||||
generatedOutputs, err := transformer.Generate(ctx, transform.Request{
|
generatedOutputs, err := transformer.Generate(ctx, transform.Request{
|
||||||
SourceBundle: req.SourceBundle,
|
SourceBundle: req.SourceBundle,
|
||||||
SourceBackend: req.SourceBackend,
|
SourceBackend: req.SourceBackend,
|
||||||
|
Markdown: transform.MarkdownOptions{
|
||||||
|
Mode: req.Transform.MarkdownToHTML.Mode,
|
||||||
|
Input: req.Transform.MarkdownToHTML.Input,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -93,25 +97,42 @@ func rejectOutputCollisions(outputs []Output) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func stateOutputs(outputs []Output) []state.OutputFile {
|
func (o Output) StateOutputFile() state.OutputFile {
|
||||||
|
return state.OutputFile{
|
||||||
|
Path: o.DestinationPath,
|
||||||
|
Kind: o.Kind,
|
||||||
|
SourcePath: o.SourcePath,
|
||||||
|
Transform: o.Transform,
|
||||||
|
URL: o.URL,
|
||||||
|
SHA256: o.SHA256,
|
||||||
|
Size: o.Size,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o Output) ManagedPath() string {
|
||||||
|
return o.DestinationPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func StateOutputFiles(outputs []Output) []state.OutputFile {
|
||||||
files := make([]state.OutputFile, 0, len(outputs))
|
files := make([]state.OutputFile, 0, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
files = append(files, state.OutputFile{
|
files = append(files, output.StateOutputFile())
|
||||||
Path: output.DestinationPath,
|
|
||||||
Kind: output.Kind,
|
|
||||||
SourcePath: output.SourcePath,
|
|
||||||
Transform: output.Transform,
|
|
||||||
SHA256: output.SHA256,
|
|
||||||
Size: output.Size,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
return files
|
return files
|
||||||
}
|
}
|
||||||
|
|
||||||
func managedOutputPaths(outputs []Output) []string {
|
func ManagedOutputPaths(outputs []Output) []string {
|
||||||
paths := make([]string, 0, len(outputs))
|
paths := make([]string, 0, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
paths = append(paths, output.DestinationPath)
|
paths = append(paths, output.ManagedPath())
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateOutputManagedPaths(outputs []state.OutputFile) []string {
|
||||||
|
paths := make([]string, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
paths = append(paths, output.Path)
|
||||||
}
|
}
|
||||||
return paths
|
return paths
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,11 +6,58 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestOutputStateProjection(t *testing.T) {
|
||||||
|
sourceOutput := Output{
|
||||||
|
SourcePath: "report.md",
|
||||||
|
DestinationPath: "report.md",
|
||||||
|
Kind: state.OutputKindSource,
|
||||||
|
URL: "https://reports.example.com/report.md",
|
||||||
|
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||||
|
Size: 123,
|
||||||
|
}
|
||||||
|
sourceState := sourceOutput.StateOutputFile()
|
||||||
|
if sourceState.Path != "report.md" || sourceState.Kind != state.OutputKindSource || sourceState.SourcePath != "report.md" || sourceState.URL != sourceOutput.URL || sourceState.SHA256 != sourceOutput.SHA256 || sourceState.Size != sourceOutput.Size {
|
||||||
|
t.Fatalf("source state output = %#v", sourceState)
|
||||||
|
}
|
||||||
|
|
||||||
|
generatedOutput := Output{
|
||||||
|
SourcePath: "report.md",
|
||||||
|
DestinationPath: "report.html",
|
||||||
|
Kind: state.OutputKindGenerated,
|
||||||
|
Transform: transform.MarkdownToHTML,
|
||||||
|
URL: "https://reports.example.com/report.html",
|
||||||
|
SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||||
|
Size: 456,
|
||||||
|
}
|
||||||
|
generatedState := generatedOutput.StateOutputFile()
|
||||||
|
if generatedState.Path != "report.html" || generatedState.Kind != state.OutputKindGenerated || generatedState.SourcePath != "report.md" || generatedState.Transform != transform.MarkdownToHTML || generatedState.URL != generatedOutput.URL || generatedState.SHA256 != generatedOutput.SHA256 || generatedState.Size != generatedOutput.Size {
|
||||||
|
t.Fatalf("generated state output = %#v", generatedState)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSliceProjections(t *testing.T) {
|
||||||
|
outputs := []Output{
|
||||||
|
{SourcePath: "report.md", DestinationPath: "report.md", Kind: state.OutputKindSource, SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Size: 1},
|
||||||
|
{SourcePath: "report.md", DestinationPath: "report.html", Kind: state.OutputKindGenerated, Transform: transform.MarkdownToHTML, URL: "https://reports.example.com/report.html", SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Size: 2},
|
||||||
|
}
|
||||||
|
|
||||||
|
stateOutputs := StateOutputFiles(outputs)
|
||||||
|
if len(stateOutputs) != 2 || stateOutputs[1].Path != "report.html" || stateOutputs[1].Transform != transform.MarkdownToHTML || stateOutputs[1].URL != outputs[1].URL {
|
||||||
|
t.Fatalf("state outputs = %#v", stateOutputs)
|
||||||
|
}
|
||||||
|
|
||||||
|
paths := ManagedOutputPaths(outputs)
|
||||||
|
if len(paths) != 2 || paths[0] != "report.md" || paths[1] != "report.html" {
|
||||||
|
t.Fatalf("managed paths = %#v", paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPlanOutputsRejectsCollision(t *testing.T) {
|
func TestPlanOutputsRejectsCollision(t *testing.T) {
|
||||||
sourceBackend := fake.New()
|
sourceBackend := fake.New()
|
||||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||||
@@ -103,6 +150,35 @@ func TestPlanOutputsUsesRegisteredTransformer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPlanOutputsPassesMarkdownOptions(t *testing.T) {
|
||||||
|
data := []byte("<p>Generated</p>\n")
|
||||||
|
transformer := &recordingTransformer{outputs: []transform.Output{{
|
||||||
|
Path: "index.html",
|
||||||
|
SourcePath: "report.md",
|
||||||
|
Transform: transform.MarkdownToHTML,
|
||||||
|
Data: data,
|
||||||
|
SHA256: bundle.FileDigest(data),
|
||||||
|
Size: int64(len(data)),
|
||||||
|
}}}
|
||||||
|
|
||||||
|
_, err := PlanOutputs(context.Background(), Request{
|
||||||
|
Publish: config.PublishPolicy{HTML: true},
|
||||||
|
Transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
Mode: config.TransformModeIndex,
|
||||||
|
Input: "report.md",
|
||||||
|
}},
|
||||||
|
Transformers: testResolver{transform.MarkdownToHTML: transformer},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanOutputs() error = %v", err)
|
||||||
|
}
|
||||||
|
if transformer.request.Markdown.Mode != config.TransformModeIndex || transformer.request.Markdown.Input != "report.md" {
|
||||||
|
t.Fatalf("markdown options = %#v, want index/report.md", transformer.request.Markdown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildRejectsHTMLWithoutTransform(t *testing.T) {
|
func TestBuildRejectsHTMLWithoutTransform(t *testing.T) {
|
||||||
sourceBackend := fake.New()
|
sourceBackend := fake.New()
|
||||||
destinationBackend := fake.New()
|
destinationBackend := fake.New()
|
||||||
@@ -148,6 +224,14 @@ func TestValidateRequestChecksPublishTransformPolicy(t *testing.T) {
|
|||||||
Mode: config.TransformModeSidecar,
|
Mode: config.TransformModeSidecar,
|
||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "html only index allowed",
|
||||||
|
publish: config.PublishPolicy{HTML: true},
|
||||||
|
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
Mode: config.TransformModeIndex,
|
||||||
|
}},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "source and html sidecar allowed",
|
name: "source and html sidecar allowed",
|
||||||
publish: config.PublishPolicy{Source: true, HTML: true},
|
publish: config.PublishPolicy{Source: true, HTML: true},
|
||||||
@@ -184,6 +268,15 @@ func TestValidateRequestChecksPublishTransformPolicy(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "source only enabled transform rejected",
|
||||||
|
publish: config.PublishPolicy{Source: true},
|
||||||
|
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
Mode: config.TransformModeSidecar,
|
||||||
|
}},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "enabled markdown wrong mode rejected",
|
name: "enabled markdown wrong mode rejected",
|
||||||
publish: config.PublishPolicy{Source: true},
|
publish: config.PublishPolicy{Source: true},
|
||||||
@@ -253,3 +346,14 @@ type testTransformer struct {
|
|||||||
func (t testTransformer) Generate(context.Context, transform.Request) ([]transform.Output, error) {
|
func (t testTransformer) Generate(context.Context, transform.Request) ([]transform.Output, error) {
|
||||||
return t.outputs, t.err
|
return t.outputs, t.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type recordingTransformer struct {
|
||||||
|
outputs []transform.Output
|
||||||
|
request transform.Request
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *recordingTransformer) Generate(_ context.Context, req transform.Request) ([]transform.Output, error) {
|
||||||
|
t.request = req
|
||||||
|
return t.outputs, t.err
|
||||||
|
}
|
||||||
|
|||||||
@@ -30,8 +30,10 @@ type Request struct {
|
|||||||
SourceBackend storage.Backend
|
SourceBackend storage.Backend
|
||||||
DestinationBackend storage.Backend
|
DestinationBackend storage.Backend
|
||||||
DestinationBundlePath string
|
DestinationBundlePath string
|
||||||
|
PathMapping string
|
||||||
Publish config.PublishPolicy
|
Publish config.PublishPolicy
|
||||||
Transform config.Transform
|
Transform config.Transform
|
||||||
|
Links *config.Links
|
||||||
Transformers TransformerResolver
|
Transformers TransformerResolver
|
||||||
Transfer config.TransferPolicy
|
Transfer config.TransferPolicy
|
||||||
DistributorVersion string
|
DistributorVersion string
|
||||||
@@ -48,9 +50,11 @@ type Plan struct {
|
|||||||
BundleID string
|
BundleID string
|
||||||
BundlePath string
|
BundlePath string
|
||||||
DestinationBundlePath string
|
DestinationBundlePath string
|
||||||
|
PathMapping string
|
||||||
Action Action
|
Action Action
|
||||||
Reason string
|
Reason string
|
||||||
Force bool
|
Force bool
|
||||||
|
PrimaryURL string
|
||||||
Outputs []Output
|
Outputs []Output
|
||||||
ExistingState *state.DistributorState
|
ExistingState *state.DistributorState
|
||||||
}
|
}
|
||||||
@@ -60,6 +64,7 @@ type Output struct {
|
|||||||
DestinationPath string
|
DestinationPath string
|
||||||
Kind string
|
Kind string
|
||||||
Transform string
|
Transform string
|
||||||
|
URL string
|
||||||
Data []byte
|
Data []byte
|
||||||
SHA256 string
|
SHA256 string
|
||||||
Size int64
|
Size int64
|
||||||
@@ -73,11 +78,15 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Plan{}, err
|
return Plan{}, err
|
||||||
}
|
}
|
||||||
|
outputs, primaryURL, err := PlanLinks(req, outputs)
|
||||||
|
if err != nil {
|
||||||
|
return Plan{}, err
|
||||||
|
}
|
||||||
status, err := inspectDestination(ctx, req.DestinationBackend, req.DestinationBundlePath)
|
status, err := inspectDestination(ctx, req.DestinationBackend, req.DestinationBundlePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Plan{}, err
|
return Plan{}, err
|
||||||
}
|
}
|
||||||
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
|
comparison := compareDestination(req, status)
|
||||||
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
|
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
|
||||||
plan := Plan{
|
plan := Plan{
|
||||||
PipelineID: req.PipelineID,
|
PipelineID: req.PipelineID,
|
||||||
@@ -85,9 +94,11 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
|||||||
BundleID: req.SourceBundle.Manifest.ID,
|
BundleID: req.SourceBundle.Manifest.ID,
|
||||||
BundlePath: req.SourceBundle.RootRelativePath,
|
BundlePath: req.SourceBundle.RootRelativePath,
|
||||||
DestinationBundlePath: req.DestinationBundlePath,
|
DestinationBundlePath: req.DestinationBundlePath,
|
||||||
|
PathMapping: req.PathMapping,
|
||||||
Action: action,
|
Action: action,
|
||||||
Reason: reason,
|
Reason: reason,
|
||||||
Force: action == ActionForceReplace,
|
Force: action == ActionForceReplace,
|
||||||
|
PrimaryURL: primaryURL,
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
ExistingState: status.State,
|
ExistingState: status.State,
|
||||||
}
|
}
|
||||||
@@ -116,6 +127,21 @@ func validateRequest(req Request) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func compareDestination(req Request, status state.DestinationStatus) state.Comparison {
|
||||||
|
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
|
||||||
|
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict || status.State == nil {
|
||||||
|
return comparison
|
||||||
|
}
|
||||||
|
destinationManifest := status.State.Source.Manifest
|
||||||
|
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
|
||||||
|
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
|
||||||
|
}
|
||||||
|
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
|
||||||
|
return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"}
|
||||||
|
}
|
||||||
|
return comparison
|
||||||
|
}
|
||||||
|
|
||||||
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
|
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
|
||||||
switch comparison.Outcome {
|
switch comparison.Outcome {
|
||||||
case state.OutcomeDestinationAbsent:
|
case state.OutcomeDestinationAbsent:
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ type DistributorState struct {
|
|||||||
DestinationID string
|
DestinationID string
|
||||||
PublishedAt time.Time
|
PublishedAt time.Time
|
||||||
Source SourceState
|
Source SourceState
|
||||||
|
Links *LinkState
|
||||||
Outputs []OutputFile
|
Outputs []OutputFile
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -26,11 +27,16 @@ type SourceState struct {
|
|||||||
Manifest bundle.Manifest
|
Manifest bundle.Manifest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type LinkState struct {
|
||||||
|
PrimaryURL string
|
||||||
|
}
|
||||||
|
|
||||||
type OutputFile struct {
|
type OutputFile struct {
|
||||||
Path string
|
Path string
|
||||||
Kind string
|
Kind string
|
||||||
SourcePath string
|
SourcePath string
|
||||||
Transform string
|
Transform string
|
||||||
|
URL string
|
||||||
SHA256 string
|
SHA256 string
|
||||||
Size int64
|
Size int64
|
||||||
}
|
}
|
||||||
@@ -42,6 +48,7 @@ type rawDistributorState struct {
|
|||||||
DestinationID *string `json:"destination_id"`
|
DestinationID *string `json:"destination_id"`
|
||||||
PublishedAt *string `json:"published_at"`
|
PublishedAt *string `json:"published_at"`
|
||||||
Source *rawSourceState `json:"source"`
|
Source *rawSourceState `json:"source"`
|
||||||
|
Links *rawLinkState `json:"links"`
|
||||||
Outputs []rawOutputFile `json:"outputs"`
|
Outputs []rawOutputFile `json:"outputs"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -49,11 +56,16 @@ type rawSourceState struct {
|
|||||||
Manifest json.RawMessage `json:"manifest"`
|
Manifest json.RawMessage `json:"manifest"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type rawLinkState struct {
|
||||||
|
PrimaryURL string `json:"primary_url"`
|
||||||
|
}
|
||||||
|
|
||||||
type rawOutputFile struct {
|
type rawOutputFile struct {
|
||||||
Path *string `json:"path"`
|
Path *string `json:"path"`
|
||||||
Kind *string `json:"kind"`
|
Kind *string `json:"kind"`
|
||||||
SourcePath *string `json:"source_path"`
|
SourcePath *string `json:"source_path"`
|
||||||
Transform string `json:"transform"`
|
Transform string `json:"transform"`
|
||||||
|
URL string `json:"url"`
|
||||||
SHA256 *string `json:"sha256"`
|
SHA256 *string `json:"sha256"`
|
||||||
Size *int64 `json:"size"`
|
Size *int64 `json:"size"`
|
||||||
}
|
}
|
||||||
@@ -112,6 +124,9 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
|||||||
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
|
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
|
||||||
}
|
}
|
||||||
state.Source.Manifest = manifest
|
state.Source.Manifest = manifest
|
||||||
|
if raw.Links != nil {
|
||||||
|
state.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
||||||
|
}
|
||||||
if raw.Outputs == nil {
|
if raw.Outputs == nil {
|
||||||
return DistributorState{}, fmt.Errorf("state outputs is required")
|
return DistributorState{}, fmt.Errorf("state outputs is required")
|
||||||
}
|
}
|
||||||
@@ -161,6 +176,7 @@ func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
|
|||||||
Kind: *raw.Kind,
|
Kind: *raw.Kind,
|
||||||
SourcePath: *raw.SourcePath,
|
SourcePath: *raw.SourcePath,
|
||||||
Transform: raw.Transform,
|
Transform: raw.Transform,
|
||||||
|
URL: raw.URL,
|
||||||
SHA256: *raw.SHA256,
|
SHA256: *raw.SHA256,
|
||||||
Size: *raw.Size,
|
Size: *raw.Size,
|
||||||
}, nil
|
}, nil
|
||||||
@@ -181,6 +197,7 @@ func (s DistributorState) MarshalJSON() ([]byte, error) {
|
|||||||
DestinationID string `json:"destination_id"`
|
DestinationID string `json:"destination_id"`
|
||||||
PublishedAt string `json:"published_at"`
|
PublishedAt string `json:"published_at"`
|
||||||
Source sourceJSON `json:"source"`
|
Source sourceJSON `json:"source"`
|
||||||
|
Links *LinkState `json:"links,omitempty"`
|
||||||
Outputs []OutputFile `json:"outputs"`
|
Outputs []OutputFile `json:"outputs"`
|
||||||
}
|
}
|
||||||
return json.Marshal(stateJSON{
|
return json.Marshal(stateJSON{
|
||||||
@@ -190,16 +207,25 @@ func (s DistributorState) MarshalJSON() ([]byte, error) {
|
|||||||
DestinationID: s.DestinationID,
|
DestinationID: s.DestinationID,
|
||||||
PublishedAt: s.PublishedAtString(),
|
PublishedAt: s.PublishedAtString(),
|
||||||
Source: sourceJSON{Manifest: s.Source.Manifest},
|
Source: sourceJSON{Manifest: s.Source.Manifest},
|
||||||
|
Links: s.Links,
|
||||||
Outputs: s.Outputs,
|
Outputs: s.Outputs,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (l LinkState) MarshalJSON() ([]byte, error) {
|
||||||
|
type linkJSON struct {
|
||||||
|
PrimaryURL string `json:"primary_url,omitempty"`
|
||||||
|
}
|
||||||
|
return json.Marshal(linkJSON{PrimaryURL: l.PrimaryURL})
|
||||||
|
}
|
||||||
|
|
||||||
func (o OutputFile) MarshalJSON() ([]byte, error) {
|
func (o OutputFile) MarshalJSON() ([]byte, error) {
|
||||||
type outputJSON struct {
|
type outputJSON struct {
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
SourcePath string `json:"source_path"`
|
SourcePath string `json:"source_path"`
|
||||||
Transform string `json:"transform,omitempty"`
|
Transform string `json:"transform,omitempty"`
|
||||||
|
URL string `json:"url,omitempty"`
|
||||||
SHA256 string `json:"sha256"`
|
SHA256 string `json:"sha256"`
|
||||||
Size int64 `json:"size"`
|
Size int64 `json:"size"`
|
||||||
}
|
}
|
||||||
@@ -208,6 +234,7 @@ func (o OutputFile) MarshalJSON() ([]byte, error) {
|
|||||||
Kind: o.Kind,
|
Kind: o.Kind,
|
||||||
SourcePath: o.SourcePath,
|
SourcePath: o.SourcePath,
|
||||||
Transform: o.Transform,
|
Transform: o.Transform,
|
||||||
|
URL: o.URL,
|
||||||
SHA256: o.SHA256,
|
SHA256: o.SHA256,
|
||||||
Size: o.Size,
|
Size: o.Size,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -29,6 +29,22 @@ func TestParseValidState(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseValidStateWithLinks(t *testing.T) {
|
||||||
|
body := strings.Replace(validStateJSON(t), `"outputs": [`, `"links": {"primary_url": "https://reports.example.com/archive/report.md"},`+"\n "+`"outputs": [`, 1)
|
||||||
|
body = strings.Replace(body, `"source_path": "report.md",`, `"source_path": "report.md",`+"\n "+`"url": "https://reports.example.com/archive/report.md",`, 1)
|
||||||
|
|
||||||
|
state, err := Parse([]byte(body))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() error = %v", err)
|
||||||
|
}
|
||||||
|
if state.Links == nil || state.Links.PrimaryURL != "https://reports.example.com/archive/report.md" {
|
||||||
|
t.Fatalf("links = %#v, want primary URL", state.Links)
|
||||||
|
}
|
||||||
|
if state.Outputs[0].URL != "https://reports.example.com/archive/report.md" {
|
||||||
|
t.Fatalf("output URL = %q", state.Outputs[0].URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
|
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
|
||||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
|
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
|
||||||
state, err := Parse([]byte(body))
|
state, err := Parse([]byte(body))
|
||||||
@@ -145,6 +161,12 @@ func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
|
|||||||
"negative size": func(s *DistributorState) {
|
"negative size": func(s *DistributorState) {
|
||||||
s.Outputs[0].Size = -1
|
s.Outputs[0].Size = -1
|
||||||
},
|
},
|
||||||
|
"invalid output url": func(s *DistributorState) {
|
||||||
|
s.Outputs[0].URL = "file:///tmp/report.md"
|
||||||
|
},
|
||||||
|
"invalid primary url": func(s *DistributorState) {
|
||||||
|
s.Links = &LinkState{PrimaryURL: "file:///tmp/report.md"}
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for name, mutate := range tests {
|
for name, mutate := range tests {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
@@ -157,6 +179,23 @@ func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateReportsURLFieldContext(t *testing.T) {
|
||||||
|
source := validManifest(t)
|
||||||
|
state := *withState(t, source, func(s *DistributorState) {
|
||||||
|
s.Links = &LinkState{PrimaryURL: "https://reports.example.com/archive#top"}
|
||||||
|
})
|
||||||
|
err := Validate(state)
|
||||||
|
assertStateErrorContains(t, err, "state links.primary_url")
|
||||||
|
assertStateErrorContains(t, err, "must not include a fragment")
|
||||||
|
|
||||||
|
state = *withState(t, source, func(s *DistributorState) {
|
||||||
|
s.Outputs[0].URL = "https://reports.example.com/archive?preview=1"
|
||||||
|
})
|
||||||
|
err = Validate(state)
|
||||||
|
assertStateErrorContains(t, err, "state outputs[0].url")
|
||||||
|
assertStateErrorContains(t, err, "must not include a query string")
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
|
func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
|
||||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "May 30"`, 1)
|
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "May 30"`, 1)
|
||||||
_, err := Parse([]byte(body))
|
_, err := Parse([]byte(body))
|
||||||
@@ -188,6 +227,36 @@ func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
|
||||||
|
source := validManifest(t)
|
||||||
|
state := DistributorState{
|
||||||
|
SchemaVersion: SchemaVersion,
|
||||||
|
PipelineID: "reports",
|
||||||
|
DestinationID: "archive",
|
||||||
|
PublishedAt: time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC),
|
||||||
|
Source: SourceState{Manifest: source},
|
||||||
|
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
||||||
|
Outputs: []OutputFile{{
|
||||||
|
Path: "report.md",
|
||||||
|
Kind: OutputKindSource,
|
||||||
|
SourcePath: "report.md",
|
||||||
|
URL: "https://reports.example.com/archive/report.md",
|
||||||
|
SHA256: source.Files[0].SHA256,
|
||||||
|
Size: source.Files[0].Size,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(state)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), `"links":{"primary_url":"https://reports.example.com/archive/report.md"}`) {
|
||||||
|
t.Fatalf("json = %s, want links primary URL", data)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), `"url":"https://reports.example.com/archive/report.md"`) {
|
||||||
|
t.Fatalf("json = %s, want output URL", data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func validStateJSON(t *testing.T) string {
|
func validStateJSON(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
return validStateWithManifestJSON(t, manifestJSON(t))
|
return validStateWithManifestJSON(t, manifestJSON(t))
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -28,6 +29,11 @@ func Validate(s DistributorState) error {
|
|||||||
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
|
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
|
||||||
return fmt.Errorf("state source.manifest: %w", err)
|
return fmt.Errorf("state source.manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
if s.Links != nil && s.Links.PrimaryURL != "" {
|
||||||
|
if err := link.ValidateHTTPURL(s.Links.PrimaryURL); err != nil {
|
||||||
|
return fmt.Errorf("state links.primary_url: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
if s.Outputs == nil {
|
if s.Outputs == nil {
|
||||||
return fmt.Errorf("state outputs is required")
|
return fmt.Errorf("state outputs is required")
|
||||||
}
|
}
|
||||||
@@ -63,6 +69,11 @@ func validateOutput(index int, output OutputFile) error {
|
|||||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
||||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||||
}
|
}
|
||||||
|
if output.URL != "" {
|
||||||
|
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
||||||
|
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package fake
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"io"
|
"io"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -133,14 +132,14 @@ func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOpti
|
|||||||
if err := storage.ValidatePrefix(prefix); err != nil {
|
if err := storage.ValidatePrefix(prefix); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
emitter := storage.NewWalkEmitter(ctx, backendName, opts, fn)
|
||||||
if entry, err := b.Stat(ctx, prefix); err == nil && entry.Type != storage.EntryTypeDirectory {
|
if entry, err := b.Stat(ctx, prefix); err == nil && entry.Type != storage.EntryTypeDirectory {
|
||||||
return emit(ctx, entry, opts, fn)
|
return storage.FinishWalk(emitter.Emit(entry))
|
||||||
} else if err != nil && !storage.IsNotFound(err) {
|
} else if err != nil && !storage.IsNotFound(err) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
entries := b.entries()
|
entries := b.entries()
|
||||||
visited := 0
|
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
if entry.Path == "" || !entryBelow(prefix, entry.Path) {
|
if entry.Path == "" || !entryBelow(prefix, entry.Path) {
|
||||||
continue
|
continue
|
||||||
@@ -148,33 +147,15 @@ func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOpti
|
|||||||
if !opts.Recursive && !isImmediateChild(prefix, entry.Path) {
|
if !opts.Recursive && !isImmediateChild(prefix, entry.Path) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if opts.Limit > 0 && visited >= opts.Limit {
|
if err := emitter.Emit(entry); err != nil {
|
||||||
return nil
|
return storage.FinishWalk(err)
|
||||||
}
|
|
||||||
visited++
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := fn(entry); err != nil {
|
|
||||||
if errors.Is(err, storage.ErrStopWalk) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return storage.NewError(storage.OpWalk, backendName, entry.Path, storage.ErrUnknown, err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||||
found := false
|
return storage.HasAny(ctx, b, prefix)
|
||||||
err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
|
|
||||||
found = true
|
|
||||||
return storage.ErrStopWalk
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return found, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||||
@@ -302,19 +283,6 @@ func (b *Backend) entries() []storage.Entry {
|
|||||||
return entries
|
return entries
|
||||||
}
|
}
|
||||||
|
|
||||||
func emit(ctx context.Context, entry storage.Entry, opts storage.WalkOptions, fn storage.WalkFunc) error {
|
|
||||||
if opts.Limit > 0 && opts.Limit < 1 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := fn(entry); err != nil && !errors.Is(err, storage.ErrStopWalk) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func entryBelow(prefix, path string) bool {
|
func entryBelow(prefix, path string) bool {
|
||||||
if prefix == "" {
|
if prefix == "" {
|
||||||
return path != ""
|
return path != ""
|
||||||
|
|||||||
63
internal/storage/walk.go
Normal file
63
internal/storage/walk.go
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
type WalkEmitter struct {
|
||||||
|
ctx context.Context
|
||||||
|
backend string
|
||||||
|
opts WalkOptions
|
||||||
|
fn WalkFunc
|
||||||
|
count int
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWalkEmitter(ctx context.Context, backend string, opts WalkOptions, fn WalkFunc) *WalkEmitter {
|
||||||
|
return &WalkEmitter{
|
||||||
|
ctx: ctx,
|
||||||
|
backend: backend,
|
||||||
|
opts: opts,
|
||||||
|
fn: fn,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *WalkEmitter) Emit(entry Entry) error {
|
||||||
|
if err := e.ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if e.opts.Limit > 0 && e.count >= e.opts.Limit {
|
||||||
|
return ErrStopWalk
|
||||||
|
}
|
||||||
|
e.count++
|
||||||
|
if err := e.fn(entry); err != nil {
|
||||||
|
if errors.Is(err, ErrStopWalk) {
|
||||||
|
return ErrStopWalk
|
||||||
|
}
|
||||||
|
return NewError(OpWalk, e.backend, entry.Path, ErrUnknown, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *WalkEmitter) LimitReached() bool {
|
||||||
|
return e.opts.Limit > 0 && e.count >= e.opts.Limit
|
||||||
|
}
|
||||||
|
|
||||||
|
func FinishWalk(err error) error {
|
||||||
|
if errors.Is(err, ErrStopWalk) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func HasAny(ctx context.Context, backend Backend, prefix string) (bool, error) {
|
||||||
|
found := false
|
||||||
|
err := backend.Walk(ctx, prefix, WalkOptions{Recursive: false, Limit: 1}, func(Entry) error {
|
||||||
|
found = true
|
||||||
|
return ErrStopWalk
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return found, nil
|
||||||
|
}
|
||||||
107
internal/storage/walk_test.go
Normal file
107
internal/storage/walk_test.go
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWalkEmitterHonorsLimit(t *testing.T) {
|
||||||
|
emitter := NewWalkEmitter(context.Background(), "test", WalkOptions{Limit: 2}, func(Entry) error {
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := emitter.Emit(Entry{Path: "one"}); err != nil {
|
||||||
|
t.Fatalf("first Emit() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := emitter.Emit(Entry{Path: "two"}); err != nil {
|
||||||
|
t.Fatalf("second Emit() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := emitter.Emit(Entry{Path: "three"}); !errors.Is(err, ErrStopWalk) {
|
||||||
|
t.Fatalf("third Emit() error = %v, want ErrStopWalk", err)
|
||||||
|
}
|
||||||
|
if !emitter.LimitReached() {
|
||||||
|
t.Fatal("LimitReached() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkEmitterStopsWithoutError(t *testing.T) {
|
||||||
|
emitter := NewWalkEmitter(context.Background(), "test", WalkOptions{}, func(Entry) error {
|
||||||
|
return ErrStopWalk
|
||||||
|
})
|
||||||
|
|
||||||
|
err := emitter.Emit(Entry{Path: "one"})
|
||||||
|
if !errors.Is(err, ErrStopWalk) {
|
||||||
|
t.Fatalf("Emit() error = %v, want ErrStopWalk", err)
|
||||||
|
}
|
||||||
|
if err := FinishWalk(err); err != nil {
|
||||||
|
t.Fatalf("FinishWalk() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkEmitterWrapsCallbackErrors(t *testing.T) {
|
||||||
|
callbackErr := errors.New("callback failed")
|
||||||
|
emitter := NewWalkEmitter(context.Background(), "test", WalkOptions{}, func(Entry) error {
|
||||||
|
return callbackErr
|
||||||
|
})
|
||||||
|
|
||||||
|
err := emitter.Emit(Entry{Path: "one"})
|
||||||
|
if !errors.Is(err, callbackErr) {
|
||||||
|
t.Fatalf("Emit() error = %v, want callback error", err)
|
||||||
|
}
|
||||||
|
var storageErr *Error
|
||||||
|
if !errors.As(err, &storageErr) {
|
||||||
|
t.Fatalf("Emit() error type = %T, want *Error", err)
|
||||||
|
}
|
||||||
|
if storageErr.Op != OpWalk || storageErr.Backend != "test" || storageErr.Path != "one" || storageErr.Kind != ErrUnknown {
|
||||||
|
t.Fatalf("wrapped error = %#v", storageErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkEmitterHonorsContextCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
called := false
|
||||||
|
emitter := NewWalkEmitter(ctx, "test", WalkOptions{}, func(Entry) error {
|
||||||
|
called = true
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
err := emitter.Emit(Entry{Path: "one"})
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Emit() error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
if called {
|
||||||
|
t.Fatal("callback was called after context cancellation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHasAnyUsesNonRecursiveLimitOneWalk(t *testing.T) {
|
||||||
|
backend := &recordingBackend{}
|
||||||
|
|
||||||
|
found, err := HasAny(context.Background(), backend, "bundle")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HasAny() error = %v", err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("HasAny() = false, want true")
|
||||||
|
}
|
||||||
|
if backend.prefix != "bundle" {
|
||||||
|
t.Fatalf("walk prefix = %q, want bundle", backend.prefix)
|
||||||
|
}
|
||||||
|
if backend.opts != (WalkOptions{Recursive: false, Limit: 1}) {
|
||||||
|
t.Fatalf("walk options = %#v, want non-recursive limit one", backend.opts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingBackend struct {
|
||||||
|
Backend
|
||||||
|
prefix string
|
||||||
|
opts WalkOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *recordingBackend) Walk(_ context.Context, prefix string, opts WalkOptions, fn WalkFunc) error {
|
||||||
|
b.prefix = prefix
|
||||||
|
b.opts = opts
|
||||||
|
return FinishWalk(fn(Entry{Path: "bundle/file.txt", Type: EntryTypeFile}))
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ package testutil
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -106,6 +107,53 @@ func WriteFakeSourceBundle(t testing.TB, backend *fake.Backend, relative string,
|
|||||||
return bundle.Bundle{RootRelativePath: relative, Manifest: manifest}
|
return bundle.Bundle{RootRelativePath: relative, Manifest: manifest}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func WriteFakeFile(t testing.TB, backend *fake.Backend, path, data string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("write fake file %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func AssertFakeFile(t testing.TB, backend *fake.Backend, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := backend.ReadFile(context.Background(), path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fake file %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if got := string(data); got != want {
|
||||||
|
t.Fatalf("fake file %s = %q, want %q", path, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func AssertFakeMissing(t testing.TB, backend *fake.Backend, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := backend.Stat(context.Background(), path); !storage.IsNotFound(err) {
|
||||||
|
t.Fatalf("fake file %s stat error = %v, want not found", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteFakeDestinationState(t testing.TB, backend *fake.Backend, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||||
|
t.Helper()
|
||||||
|
destinationState := DestinationState(manifest, opts)
|
||||||
|
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal destination state: %v", err)
|
||||||
|
}
|
||||||
|
statePath, err := storage.StatePath(relative)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("state path: %v", err)
|
||||||
|
}
|
||||||
|
WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||||
|
for _, output := range destinationState.Outputs {
|
||||||
|
path, err := storage.Join(relative, output.Path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("join output path: %v", err)
|
||||||
|
}
|
||||||
|
WriteFakeFile(t, backend, path, "old")
|
||||||
|
}
|
||||||
|
return destinationState
|
||||||
|
}
|
||||||
|
|
||||||
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
|
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
return writeConfigFile(t, `
|
return writeConfigFile(t, `
|
||||||
@@ -139,6 +187,136 @@ pipelines:
|
|||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func WriteLocalConfigWithPublishPolicy(t testing.TB, sourceRoot, destinationRoot string, publishSource, publishHTML bool) string {
|
||||||
|
t.Helper()
|
||||||
|
transformConfig := ""
|
||||||
|
if publishHTML {
|
||||||
|
transformConfig = `
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: sidecar`
|
||||||
|
}
|
||||||
|
return writeConfigFile(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
publish:
|
||||||
|
source: `+fmt.Sprintf("%t", publishSource)+`
|
||||||
|
html: `+fmt.Sprintf("%t", publishHTML)+transformConfig+`
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteLocalConfigWithPathMapping(t testing.TB, sourceRoot, destinationRoot, mode string) string {
|
||||||
|
t.Helper()
|
||||||
|
return writeConfigFile(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
path_mapping:
|
||||||
|
mode: `+mode+`
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteLocalConfigWithLinks(t testing.TB, sourceRoot, destinationRoot, pathMapping, baseURL, primary string, publishSource, publishHTML bool, transformMode string) string {
|
||||||
|
t.Helper()
|
||||||
|
transformConfig := ""
|
||||||
|
if publishHTML {
|
||||||
|
transformConfig = `
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: ` + transformMode
|
||||||
|
}
|
||||||
|
return writeConfigFile(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
path_mapping:
|
||||||
|
mode: `+pathMapping+`
|
||||||
|
links:
|
||||||
|
base_url: `+baseURL+`
|
||||||
|
primary: `+primary+`
|
||||||
|
publish:
|
||||||
|
source: `+fmt.Sprintf("%t", publishSource)+`
|
||||||
|
html: `+fmt.Sprintf("%t", publishHTML)+transformConfig+`
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteLocalConfigWithMarkdownTransform(t testing.TB, sourceRoot, destinationRoot string, publishSource, publishHTML bool, mode, input string) string {
|
||||||
|
t.Helper()
|
||||||
|
enabled := publishHTML
|
||||||
|
inputConfig := ""
|
||||||
|
if input != "" {
|
||||||
|
inputConfig = `
|
||||||
|
input: ` + input
|
||||||
|
}
|
||||||
|
return writeConfigFile(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
publish:
|
||||||
|
source: `+fmt.Sprintf("%t", publishSource)+`
|
||||||
|
html: `+fmt.Sprintf("%t", publishHTML)+`
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: `+fmt.Sprintf("%t", enabled)+`
|
||||||
|
mode: `+mode+inputConfig+`
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteMixedPolicyFanoutLocalConfig(t testing.TB, sourceRoot, archiveDestination, htmlDestination string) string {
|
||||||
|
t.Helper()
|
||||||
|
return writeConfigFile(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+archiveDestination+`
|
||||||
|
publish:
|
||||||
|
source: true
|
||||||
|
html: false
|
||||||
|
- id: html
|
||||||
|
backend: local
|
||||||
|
path: `+htmlDestination+`
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: sidecar
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
func WriteDestinationState(t testing.TB, root, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
func WriteDestinationState(t testing.TB, root, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
|
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
|
||||||
@@ -175,6 +353,28 @@ func ReadDestinationState(t testing.TB, path string) state.DistributorState {
|
|||||||
return destinationState
|
return destinationState
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func AssertFile(t testing.TB, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read file %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if got := string(data); got != want {
|
||||||
|
t.Fatalf("%s = %q, want %q", path, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func AssertFileContains(t testing.TB, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read file %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), want) {
|
||||||
|
t.Fatalf("%s = %q, want substring %q", path, data, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func sourceFiles(opts BundleOptions) []SourceFile {
|
func sourceFiles(opts BundleOptions) []SourceFile {
|
||||||
files := opts.Files
|
files := opts.Files
|
||||||
if files == nil {
|
if files == nil {
|
||||||
|
|||||||
@@ -25,24 +25,26 @@ func (t *Transformer) Generate(ctx context.Context, req transform.Request) ([]tr
|
|||||||
if t.renderer == nil {
|
if t.renderer == nil {
|
||||||
t.renderer = goldmark.New()
|
t.renderer = goldmark.New()
|
||||||
}
|
}
|
||||||
|
switch markdownMode(req.Markdown.Mode) {
|
||||||
|
case transform.MarkdownModeSidecar:
|
||||||
|
return t.generateSidecars(ctx, req)
|
||||||
|
case transform.MarkdownModeIndex:
|
||||||
|
return t.generateIndex(ctx, req)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("markdown mode %q is not supported", req.Markdown.Mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Transformer) generateSidecars(ctx context.Context, req transform.Request) ([]transform.Output, error) {
|
||||||
var outputs []transform.Output
|
var outputs []transform.Output
|
||||||
for _, file := range req.SourceBundle.Manifest.Files {
|
for _, file := range req.SourceBundle.Manifest.Files {
|
||||||
if !strings.HasSuffix(file.Path, ".md") {
|
if !strings.HasSuffix(file.Path, ".md") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, file.Path)
|
html, err := t.render(ctx, req, file.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
data, err := req.SourceBackend.ReadFile(ctx, sourcePath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read markdown source %q: %w", file.Path, err)
|
|
||||||
}
|
|
||||||
var rendered bytes.Buffer
|
|
||||||
if err := t.renderer.Convert(data, &rendered); err != nil {
|
|
||||||
return nil, fmt.Errorf("render markdown source %q: %w", file.Path, err)
|
|
||||||
}
|
|
||||||
html := wrapHTML(rendered.Bytes())
|
|
||||||
outputPath := strings.TrimSuffix(file.Path, ".md") + ".html"
|
outputPath := strings.TrimSuffix(file.Path, ".md") + ".html"
|
||||||
outputs = append(outputs, transform.Output{
|
outputs = append(outputs, transform.Output{
|
||||||
Path: outputPath,
|
Path: outputPath,
|
||||||
@@ -55,3 +57,78 @@ func (t *Transformer) Generate(ctx context.Context, req transform.Request) ([]tr
|
|||||||
}
|
}
|
||||||
return outputs, nil
|
return outputs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *Transformer) generateIndex(ctx context.Context, req transform.Request) ([]transform.Output, error) {
|
||||||
|
input, err := selectIndexInput(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
html, err := t.render(ctx, req, input)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []transform.Output{{
|
||||||
|
Path: "index.html",
|
||||||
|
SourcePath: input,
|
||||||
|
Transform: transform.MarkdownToHTML,
|
||||||
|
Data: html,
|
||||||
|
SHA256: bundle.FileDigest(html),
|
||||||
|
Size: int64(len(html)),
|
||||||
|
}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectIndexInput(req transform.Request) (string, error) {
|
||||||
|
if req.Markdown.Input != "" {
|
||||||
|
if err := storage.ValidatePath(req.Markdown.Input); err != nil {
|
||||||
|
return "", fmt.Errorf("markdown input %q: %w", req.Markdown.Input, err)
|
||||||
|
}
|
||||||
|
for _, file := range req.SourceBundle.Manifest.Files {
|
||||||
|
if file.Path != req.Markdown.Input {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(file.Path, ".md") {
|
||||||
|
return "", fmt.Errorf("markdown input %q must end in .md", req.Markdown.Input)
|
||||||
|
}
|
||||||
|
return file.Path, nil
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("markdown input %q is not listed in the source manifest", req.Markdown.Input)
|
||||||
|
}
|
||||||
|
|
||||||
|
var markdownFiles []string
|
||||||
|
for _, file := range req.SourceBundle.Manifest.Files {
|
||||||
|
if strings.HasSuffix(file.Path, ".md") {
|
||||||
|
markdownFiles = append(markdownFiles, file.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch len(markdownFiles) {
|
||||||
|
case 0:
|
||||||
|
return "", fmt.Errorf("markdown index mode requires one markdown source file or transform.markdown_to_html.input")
|
||||||
|
case 1:
|
||||||
|
return markdownFiles[0], nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("markdown index mode found multiple markdown source files; set transform.markdown_to_html.input")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Transformer) render(ctx context.Context, req transform.Request, sourceFile string) ([]byte, error) {
|
||||||
|
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, sourceFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
data, err := req.SourceBackend.ReadFile(ctx, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read markdown source %q: %w", sourceFile, err)
|
||||||
|
}
|
||||||
|
var rendered bytes.Buffer
|
||||||
|
if err := t.renderer.Convert(data, &rendered); err != nil {
|
||||||
|
return nil, fmt.Errorf("render markdown source %q: %w", sourceFile, err)
|
||||||
|
}
|
||||||
|
return wrapHTML(rendered.Bytes()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func markdownMode(mode string) string {
|
||||||
|
if mode == "" {
|
||||||
|
return transform.MarkdownModeSidecar
|
||||||
|
}
|
||||||
|
return mode
|
||||||
|
}
|
||||||
|
|||||||
@@ -40,6 +40,124 @@ func TestGenerateMarkdownSidecar(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGenerateMarkdownIndexExplicitInput(t *testing.T) {
|
||||||
|
backend := fake.New()
|
||||||
|
sourceBundle := testutil.WriteFakeSourceBundle(t, backend, "", testutil.BundleOptions{
|
||||||
|
ID: "bundle",
|
||||||
|
Files: []testutil.SourceFile{
|
||||||
|
{Path: "report.md", Data: "# Report\n"},
|
||||||
|
{Path: "notes.md", Data: "# Notes\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
outputs, err := New().Generate(context.Background(), transform.Request{
|
||||||
|
SourceBackend: backend,
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex, Input: "notes.md"},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := len(outputs), 1; got != want {
|
||||||
|
t.Fatalf("output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
output := outputs[0]
|
||||||
|
if output.Path != "index.html" || output.SourcePath != "notes.md" || output.Transform != transform.MarkdownToHTML {
|
||||||
|
t.Fatalf("output metadata = %#v, want index from notes.md", output)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(output.Data), "<h1>Notes</h1>") {
|
||||||
|
t.Fatalf("html = %q, want notes content", output.Data)
|
||||||
|
}
|
||||||
|
if output.SHA256 != bundle.FileDigest(output.Data) || output.Size != int64(len(output.Data)) {
|
||||||
|
t.Fatalf("digest/size metadata = %s/%d", output.SHA256, output.Size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateMarkdownIndexSelectsOnlyMarkdownFile(t *testing.T) {
|
||||||
|
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
|
||||||
|
|
||||||
|
outputs, err := New().Generate(context.Background(), transform.Request{
|
||||||
|
SourceBackend: backend,
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := outputs[0].Path, "index.html"; got != want {
|
||||||
|
t.Fatalf("path = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got, want := outputs[0].SourcePath, "report.md"; got != want {
|
||||||
|
t.Fatalf("source path = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateMarkdownIndexRejectsAmbiguousInput(t *testing.T) {
|
||||||
|
backend := fake.New()
|
||||||
|
sourceBundle := testutil.WriteFakeSourceBundle(t, backend, "", testutil.BundleOptions{
|
||||||
|
ID: "bundle",
|
||||||
|
Files: []testutil.SourceFile{
|
||||||
|
{Path: "report.md", Data: "# Report\n"},
|
||||||
|
{Path: "notes.md", Data: "# Notes\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := New().Generate(context.Background(), transform.Request{
|
||||||
|
SourceBackend: backend,
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "multiple markdown source files") {
|
||||||
|
t.Fatalf("Generate() error = %v, want ambiguous input error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateMarkdownIndexRejectsMissingMarkdown(t *testing.T) {
|
||||||
|
backend := fake.New()
|
||||||
|
sourceBundle := testutil.WriteFakeSourceBundle(t, backend, "", testutil.BundleOptions{
|
||||||
|
ID: "bundle",
|
||||||
|
Files: []testutil.SourceFile{{Path: "summary.txt", Data: "Summary\n"}},
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := New().Generate(context.Background(), transform.Request{
|
||||||
|
SourceBackend: backend,
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "requires one markdown source file") {
|
||||||
|
t.Fatalf("Generate() error = %v, want missing markdown error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateMarkdownIndexRejectsInvalidExplicitInput(t *testing.T) {
|
||||||
|
backend, sourceBundle := markdownFixture(t, "# Title\n")
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{name: "unsafe", input: "../report.md", wantError: "markdown input"},
|
||||||
|
{name: "not listed", input: "missing.md", wantError: "not listed"},
|
||||||
|
{name: "not markdown", input: "summary.txt", wantError: "must end in .md"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := New().Generate(context.Background(), transform.Request{
|
||||||
|
SourceBackend: backend,
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex, Input: tt.input},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantError) {
|
||||||
|
t.Fatalf("Generate() error = %v, want substring %q", err, tt.wantError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGenerateIgnoresNonMarkdown(t *testing.T) {
|
func TestGenerateIgnoresNonMarkdown(t *testing.T) {
|
||||||
backend := fake.New()
|
backend := fake.New()
|
||||||
if _, err := backend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil {
|
if _, err := backend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil {
|
||||||
|
|||||||
@@ -1,3 +1,8 @@
|
|||||||
package transform
|
package transform
|
||||||
|
|
||||||
const MarkdownToHTML = "markdown_to_html"
|
const MarkdownToHTML = "markdown_to_html"
|
||||||
|
|
||||||
|
const (
|
||||||
|
MarkdownModeSidecar = "sidecar"
|
||||||
|
MarkdownModeIndex = "index"
|
||||||
|
)
|
||||||
|
|||||||
@@ -19,6 +19,12 @@ type Output struct {
|
|||||||
type Request struct {
|
type Request struct {
|
||||||
SourceBundle bundle.Bundle
|
SourceBundle bundle.Bundle
|
||||||
SourceBackend storage.Backend
|
SourceBackend storage.Backend
|
||||||
|
Markdown MarkdownOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
type MarkdownOptions struct {
|
||||||
|
Mode string
|
||||||
|
Input string
|
||||||
}
|
}
|
||||||
|
|
||||||
type Transformer interface {
|
type Transformer interface {
|
||||||
|
|||||||
118
pkg/bundle/build.go
Normal file
118
pkg/bundle/build.go
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func BuildManifest(opts BuildOptions) (Manifest, error) {
|
||||||
|
if opts.Root == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("root is required")
|
||||||
|
}
|
||||||
|
if opts.ID == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("id is required")
|
||||||
|
}
|
||||||
|
explicit := len(opts.Files) > 0
|
||||||
|
if explicit == opts.Scan {
|
||||||
|
return Manifest{}, fmt.Errorf("select exactly one file mode")
|
||||||
|
}
|
||||||
|
created := opts.Created
|
||||||
|
if created.IsZero() {
|
||||||
|
created = time.Now().UTC()
|
||||||
|
}
|
||||||
|
|
||||||
|
paths := append([]string(nil), opts.Files...)
|
||||||
|
var err error
|
||||||
|
if opts.Scan {
|
||||||
|
paths, err = scanSourcePaths(opts.Root)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
files := make([]ManifestFile, 0, len(paths))
|
||||||
|
for _, sourcePath := range paths {
|
||||||
|
file, err := buildManifestFile(opts.Root, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
files = append(files, file)
|
||||||
|
}
|
||||||
|
manifest := Manifest{
|
||||||
|
SchemaVersion: SchemaVersion,
|
||||||
|
ID: opts.ID,
|
||||||
|
Created: created,
|
||||||
|
Files: files,
|
||||||
|
}
|
||||||
|
manifest.Digest = BundleDigest(manifest.Files)
|
||||||
|
if err := ValidateManifest(manifest); err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
return manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanSourcePaths(root string) ([]string, error) {
|
||||||
|
var paths []string
|
||||||
|
err := filepath.WalkDir(root, func(filePath string, entry os.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if filePath == root {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(root, filePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sourcePath := filepath.ToSlash(relative)
|
||||||
|
if entry.Type()&os.ModeSymlink != 0 {
|
||||||
|
return fmt.Errorf("source file %q must be a regular file", sourcePath)
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if path.Base(sourcePath) == ManifestName || path.Base(sourcePath) == distributorStateName {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
info, err := entry.Info()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("source file %q must be a regular file", sourcePath)
|
||||||
|
}
|
||||||
|
paths = append(paths, sourcePath)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(paths)
|
||||||
|
return paths, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildManifestFile(root, sourcePath string) (ManifestFile, error) {
|
||||||
|
if err := ValidateSourcePath(sourcePath); err != nil {
|
||||||
|
return ManifestFile{}, fmt.Errorf("source file %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
fullPath := filepath.Join(root, filepath.FromSlash(sourcePath))
|
||||||
|
info, err := os.Lstat(fullPath)
|
||||||
|
if err != nil {
|
||||||
|
return ManifestFile{}, fmt.Errorf("source file %q stat: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return ManifestFile{}, fmt.Errorf("source file %q must be a regular file", sourcePath)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(fullPath)
|
||||||
|
if err != nil {
|
||||||
|
return ManifestFile{}, fmt.Errorf("source file %q read: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
return ManifestFile{
|
||||||
|
Path: sourcePath,
|
||||||
|
SHA256: FileDigest(data),
|
||||||
|
Size: int64(len(data)),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
363
pkg/bundle/bundle_test.go
Normal file
363
pkg/bundle/bundle_test.go
Normal file
@@ -0,0 +1,363 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildManifestExplicitFilesPreservesOrder(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "b.txt", "bravo")
|
||||||
|
writeFile(t, root, "a.txt", "alpha")
|
||||||
|
before := time.Now().UTC()
|
||||||
|
|
||||||
|
manifest, err := BuildManifest(BuildOptions{
|
||||||
|
Root: root,
|
||||||
|
ID: "reports.example",
|
||||||
|
Files: []string{"b.txt", "a.txt"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BuildManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
after := time.Now().UTC()
|
||||||
|
|
||||||
|
paths := manifestPaths(manifest)
|
||||||
|
if want := []string{"b.txt", "a.txt"}; !reflect.DeepEqual(paths, want) {
|
||||||
|
t.Fatalf("paths = %v, want %v", paths, want)
|
||||||
|
}
|
||||||
|
if manifest.SchemaVersion != SchemaVersion {
|
||||||
|
t.Fatalf("schema version = %d, want %d", manifest.SchemaVersion, SchemaVersion)
|
||||||
|
}
|
||||||
|
if manifest.ID != "reports.example" {
|
||||||
|
t.Fatalf("id = %q", manifest.ID)
|
||||||
|
}
|
||||||
|
if manifest.Created.Before(before) || manifest.Created.After(after) {
|
||||||
|
t.Fatalf("created = %s, want between %s and %s", manifest.Created, before, after)
|
||||||
|
}
|
||||||
|
if err := ValidateManifest(manifest); err != nil {
|
||||||
|
t.Fatalf("ValidateManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := ValidateBundle(root, manifest); err != nil {
|
||||||
|
t.Fatalf("ValidateBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildManifestScanSortsAndFiltersMetadata(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "z.txt", "zulu")
|
||||||
|
writeFile(t, root, "nested/.hidden", "hidden")
|
||||||
|
writeFile(t, root, ManifestName, "old manifest")
|
||||||
|
writeFile(t, root, distributorStateName, "state")
|
||||||
|
writeFile(t, root, "nested/manifest.json", "nested manifest")
|
||||||
|
writeFile(t, root, "nested/.distributor.json", "nested state")
|
||||||
|
created := time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
manifest, err := BuildManifest(BuildOptions{
|
||||||
|
Root: root,
|
||||||
|
ID: "reports.scan",
|
||||||
|
Created: created,
|
||||||
|
Scan: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BuildManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !manifest.Created.Equal(created) {
|
||||||
|
t.Fatalf("created = %s, want %s", manifest.Created, created)
|
||||||
|
}
|
||||||
|
paths := manifestPaths(manifest)
|
||||||
|
if want := []string{"nested/.hidden", "z.txt"}; !reflect.DeepEqual(paths, want) {
|
||||||
|
t.Fatalf("paths = %v, want %v", paths, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildManifestRequiresOneFileMode(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "report.txt", "report")
|
||||||
|
|
||||||
|
tests := []BuildOptions{
|
||||||
|
{Root: root, ID: "reports.none"},
|
||||||
|
{Root: root, ID: "reports.both", Files: []string{"report.txt"}, Scan: true},
|
||||||
|
}
|
||||||
|
for _, opts := range tests {
|
||||||
|
if _, err := BuildManifest(opts); err == nil {
|
||||||
|
t.Fatalf("BuildManifest(%+v) error = nil, want error", opts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildManifestRejectsUnsafePath(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "report.txt", "report")
|
||||||
|
|
||||||
|
_, err := BuildManifest(BuildOptions{
|
||||||
|
Root: root,
|
||||||
|
ID: "reports.unsafe",
|
||||||
|
Files: []string{"../report.txt"},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("BuildManifest() error = nil, want unsafe path error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildManifestRejectsSymlink(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "target.txt", "target")
|
||||||
|
if err := os.Symlink("target.txt", filepath.Join(root, "link.txt")); err != nil {
|
||||||
|
t.Skipf("symlink unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := BuildManifest(BuildOptions{
|
||||||
|
Root: root,
|
||||||
|
ID: "reports.symlink",
|
||||||
|
Files: []string{"link.txt"},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "regular file") {
|
||||||
|
t.Fatalf("BuildManifest() error = %v, want regular file error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScanRejectsSymlink(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "target.txt", "target")
|
||||||
|
if err := os.Symlink("target.txt", filepath.Join(root, "link.txt")); err != nil {
|
||||||
|
t.Skipf("symlink unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := BuildManifest(BuildOptions{
|
||||||
|
Root: root,
|
||||||
|
ID: "reports.scan.symlink",
|
||||||
|
Scan: true,
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "regular file") {
|
||||||
|
t.Fatalf("BuildManifest() error = %v, want regular file error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDigestFunctionsUseCanonicalFilePayload(t *testing.T) {
|
||||||
|
files := []ManifestFile{
|
||||||
|
{Path: "report.md", SHA256: FileDigest([]byte("report")), Size: 6},
|
||||||
|
{Path: "summary.txt", SHA256: FileDigest([]byte("summary")), Size: 7},
|
||||||
|
}
|
||||||
|
payload := CanonicalFilePayload(files)
|
||||||
|
if want := `[{"path":"report.md","sha256":"sha256:845e91831319e89c4d656bdb80c278ac09a7230d61e5dfd2e1b1fbb436ac8917","size":6},{"path":"summary.txt","sha256":"sha256:761b7ad8ad439b2855fcbb611331c646ef0870b0631247bba3f3025cb6df5a53","size":7}]`; payload != want {
|
||||||
|
t.Fatalf("payload = %q, want %q", payload, want)
|
||||||
|
}
|
||||||
|
if digest := BundleDigest(files); !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 {
|
||||||
|
t.Fatalf("BundleDigest() = %q, want sha256 digest", digest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseMarshalLoadAndWriteManifest(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "report.txt", "report")
|
||||||
|
manifest, err := BuildManifest(BuildOptions{
|
||||||
|
Root: root,
|
||||||
|
ID: "reports.json",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: []string{"report.txt"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BuildManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
data, err := MarshalManifest(manifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MarshalManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(string(data), "\n") {
|
||||||
|
t.Fatalf("manifest JSON = %q, want trailing newline", string(data))
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), `"schema_version": 1`) || !strings.Contains(string(data), `"sha256": "`) {
|
||||||
|
t.Fatalf("manifest JSON = %q, want fixed manifest fields", string(data))
|
||||||
|
}
|
||||||
|
parsed, err := ParseManifest(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(parsed, manifest) {
|
||||||
|
t.Fatalf("parsed manifest = %#v, want %#v", parsed, manifest)
|
||||||
|
}
|
||||||
|
if err := WriteManifest(root, manifest, WriteManifestOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
loaded, err := LoadManifest(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(loaded, manifest) {
|
||||||
|
t.Fatalf("loaded manifest = %#v, want %#v", loaded, manifest)
|
||||||
|
}
|
||||||
|
if err := WriteManifest(root, manifest, WriteManifestOptions{}); err == nil {
|
||||||
|
t.Fatal("WriteManifest() error = nil, want exists error")
|
||||||
|
}
|
||||||
|
manifest.ID = "reports.updated"
|
||||||
|
manifest.Digest = BundleDigest(manifest.Files)
|
||||||
|
if err := WriteManifest(root, manifest, WriteManifestOptions{Overwrite: true}); err != nil {
|
||||||
|
t.Fatalf("WriteManifest(overwrite) error = %v", err)
|
||||||
|
}
|
||||||
|
loaded, err = LoadManifest(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadManifest() after overwrite error = %v", err)
|
||||||
|
}
|
||||||
|
if loaded.ID != "reports.updated" {
|
||||||
|
t.Fatalf("loaded id = %q, want updated", loaded.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateManifestRejectsInvalidManifest(t *testing.T) {
|
||||||
|
created := time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC)
|
||||||
|
fileDigest := FileDigest([]byte("report"))
|
||||||
|
valid := Manifest{
|
||||||
|
SchemaVersion: SchemaVersion,
|
||||||
|
ID: "reports.valid",
|
||||||
|
Created: created,
|
||||||
|
Files: []ManifestFile{{Path: "report.txt", SHA256: fileDigest, Size: 6}},
|
||||||
|
}
|
||||||
|
valid.Digest = BundleDigest(valid.Files)
|
||||||
|
|
||||||
|
tests := map[string]func(Manifest) Manifest{
|
||||||
|
"schema version": func(manifest Manifest) Manifest {
|
||||||
|
manifest.SchemaVersion = 2
|
||||||
|
return manifest
|
||||||
|
},
|
||||||
|
"id": func(manifest Manifest) Manifest {
|
||||||
|
manifest.ID = ""
|
||||||
|
return manifest
|
||||||
|
},
|
||||||
|
"created": func(manifest Manifest) Manifest {
|
||||||
|
manifest.Created = time.Time{}
|
||||||
|
return manifest
|
||||||
|
},
|
||||||
|
"files": func(manifest Manifest) Manifest {
|
||||||
|
manifest.Files = nil
|
||||||
|
manifest.Digest = BundleDigest(manifest.Files)
|
||||||
|
return manifest
|
||||||
|
},
|
||||||
|
"path": func(manifest Manifest) Manifest {
|
||||||
|
manifest.Files[0].Path = "manifest.json"
|
||||||
|
manifest.Digest = BundleDigest(manifest.Files)
|
||||||
|
return manifest
|
||||||
|
},
|
||||||
|
"duplicate": func(manifest Manifest) Manifest {
|
||||||
|
manifest.Files = append(manifest.Files, manifest.Files[0])
|
||||||
|
manifest.Digest = BundleDigest(manifest.Files)
|
||||||
|
return manifest
|
||||||
|
},
|
||||||
|
"size": func(manifest Manifest) Manifest {
|
||||||
|
manifest.Files[0].Size = -1
|
||||||
|
manifest.Digest = BundleDigest(manifest.Files)
|
||||||
|
return manifest
|
||||||
|
},
|
||||||
|
"digest": func(manifest Manifest) Manifest {
|
||||||
|
manifest.Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
||||||
|
return manifest
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, mutate := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if err := ValidateManifest(mutate(valid)); err == nil {
|
||||||
|
t.Fatal("ValidateManifest() error = nil, want error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateBundleRejectsLocalFileProblems(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "report.txt", "report")
|
||||||
|
manifest, err := BuildManifest(BuildOptions{
|
||||||
|
Root: root,
|
||||||
|
ID: "reports.bundle",
|
||||||
|
Files: []string{"report.txt"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BuildManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := ValidateBundle(root, manifest); err != nil {
|
||||||
|
t.Fatalf("ValidateBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
missing := cloneManifest(manifest)
|
||||||
|
missing.Files[0].Path = "missing.txt"
|
||||||
|
missing.Digest = BundleDigest(missing.Files)
|
||||||
|
if err := ValidateBundle(root, missing); err == nil {
|
||||||
|
t.Fatal("ValidateBundle() missing error = nil, want error")
|
||||||
|
}
|
||||||
|
|
||||||
|
sizeMismatch := cloneManifest(manifest)
|
||||||
|
sizeMismatch.Files[0].Size++
|
||||||
|
sizeMismatch.Digest = BundleDigest(sizeMismatch.Files)
|
||||||
|
if err := ValidateBundle(root, sizeMismatch); err == nil || !strings.Contains(err.Error(), "size mismatch") {
|
||||||
|
t.Fatalf("ValidateBundle() size error = %v, want size mismatch", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
digestMismatch := cloneManifest(manifest)
|
||||||
|
writeFile(t, root, "report.txt", "change")
|
||||||
|
if err := ValidateBundle(root, digestMismatch); err == nil || !strings.Contains(err.Error(), "sha256 mismatch") {
|
||||||
|
t.Fatalf("ValidateBundle() digest error = %v, want digest mismatch", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateBundleRejectsSymlink(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, root, "target.txt", "target")
|
||||||
|
if err := os.Symlink("target.txt", filepath.Join(root, "link.txt")); err != nil {
|
||||||
|
t.Skipf("symlink unavailable: %v", err)
|
||||||
|
}
|
||||||
|
file := ManifestFile{Path: "link.txt", SHA256: FileDigest([]byte("target")), Size: 6}
|
||||||
|
manifest := Manifest{
|
||||||
|
SchemaVersion: SchemaVersion,
|
||||||
|
ID: "reports.link",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: []ManifestFile{file},
|
||||||
|
}
|
||||||
|
manifest.Digest = BundleDigest(manifest.Files)
|
||||||
|
|
||||||
|
err := ValidateBundle(root, manifest)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "regular file") {
|
||||||
|
t.Fatalf("ValidateBundle() error = %v, want regular file error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateSourcePath(t *testing.T) {
|
||||||
|
valid := []string{"report.md", "nested/report.md", ".well-known/report.txt"}
|
||||||
|
for _, path := range valid {
|
||||||
|
if err := ValidateSourcePath(path); err != nil {
|
||||||
|
t.Fatalf("ValidateSourcePath(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
invalid := []string{"", "../report.md", "/report.md", "nested/../report.md", `nested\report.md`, ManifestName, distributorStateName}
|
||||||
|
for _, path := range invalid {
|
||||||
|
if err := ValidateSourcePath(path); err == nil {
|
||||||
|
t.Fatalf("ValidateSourcePath(%q) error = nil, want error", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestPaths(manifest Manifest) []string {
|
||||||
|
paths := make([]string, 0, len(manifest.Files))
|
||||||
|
for _, file := range manifest.Files {
|
||||||
|
paths = append(paths, file.Path)
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneManifest(manifest Manifest) Manifest {
|
||||||
|
manifest.Files = append([]ManifestFile(nil), manifest.Files...)
|
||||||
|
return manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFile(t *testing.T, root, relative, body string) {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(root, filepath.FromSlash(relative))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
50
pkg/bundle/digest.go
Normal file
50
pkg/bundle/digest.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||||
|
|
||||||
|
// ValidateDigest reports whether value uses the lowercase sha256:<64 hex> form.
|
||||||
|
func ValidateDigest(value string) error {
|
||||||
|
if !digestPattern.MatchString(value) {
|
||||||
|
return fmt.Errorf("must be lowercase sha256:<64 hex>")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func FileDigest(data []byte) string {
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
return "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func BundleDigest(files []ManifestFile) string {
|
||||||
|
canonical := CanonicalFilePayload(files)
|
||||||
|
sum := sha256.Sum256([]byte(canonical))
|
||||||
|
return "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func CanonicalFilePayload(files []ManifestFile) string {
|
||||||
|
var builder strings.Builder
|
||||||
|
builder.WriteByte('[')
|
||||||
|
for index, file := range files {
|
||||||
|
if index > 0 {
|
||||||
|
builder.WriteByte(',')
|
||||||
|
}
|
||||||
|
builder.WriteString(`{"path":`)
|
||||||
|
builder.WriteString(strconv.Quote(file.Path))
|
||||||
|
builder.WriteString(`,"sha256":`)
|
||||||
|
builder.WriteString(strconv.Quote(file.SHA256))
|
||||||
|
builder.WriteString(`,"size":`)
|
||||||
|
builder.WriteString(strconv.FormatInt(file.Size, 10))
|
||||||
|
builder.WriteByte('}')
|
||||||
|
}
|
||||||
|
builder.WriteByte(']')
|
||||||
|
return builder.String()
|
||||||
|
}
|
||||||
58
pkg/bundle/digest_test.go
Normal file
58
pkg/bundle/digest_test.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestValidateDigest(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid",
|
||||||
|
value: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "uppercase hex",
|
||||||
|
value: "sha256:0123456789ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing prefix",
|
||||||
|
value: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "wrong algorithm",
|
||||||
|
value: "sha512:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "short hex",
|
||||||
|
value: "sha256:0123456789abcdef",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "long hex",
|
||||||
|
value: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non hex",
|
||||||
|
value: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdeg",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
err := ValidateDigest(test.value)
|
||||||
|
if test.wantErr && err == nil {
|
||||||
|
t.Fatal("ValidateDigest() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !test.wantErr && err != nil {
|
||||||
|
t.Fatalf("ValidateDigest() error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
9
pkg/bundle/doc.go
Normal file
9
pkg/bundle/doc.go
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
// Package bundle provides producer-facing helpers for distributor source
|
||||||
|
// bundle manifests.
|
||||||
|
//
|
||||||
|
// A source bundle is a local directory containing a manifest.json file and the
|
||||||
|
// files listed by that manifest. This package owns the public manifest model,
|
||||||
|
// digest calculation, path validation, manifest parsing, manifest building,
|
||||||
|
// local bundle writing, and local bundle validation used by Go producer
|
||||||
|
// applications.
|
||||||
|
package bundle
|
||||||
73
pkg/bundle/example_test.go
Normal file
73
pkg/bundle/example_test.go
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
package bundle_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ExampleBuildManifest() {
|
||||||
|
root, err := os.MkdirTemp("", "distributor-bundle-*")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(root)
|
||||||
|
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "report.txt"), []byte("report\n"), 0o600); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest, err := bundle.BuildManifest(bundle.BuildOptions{
|
||||||
|
Root: root,
|
||||||
|
ID: "reports.example",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: []string{"report.txt"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(manifest.ID)
|
||||||
|
fmt.Println(manifest.Files[0].Path)
|
||||||
|
fmt.Println(manifest.Files[0].Size)
|
||||||
|
// Output:
|
||||||
|
// reports.example
|
||||||
|
// report.txt
|
||||||
|
// 7
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExampleWriteBundle() {
|
||||||
|
sourceRoot, err := os.MkdirTemp("", "distributor-source-*")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(sourceRoot)
|
||||||
|
outputRoot := filepath.Join(os.TempDir(), "distributor-bundle-example")
|
||||||
|
defer os.RemoveAll(outputRoot)
|
||||||
|
|
||||||
|
if err := os.WriteFile(filepath.Join(sourceRoot, "report.txt"), []byte("report\n"), 0o600); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.example",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: []bundle.BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.txt"), Path: "report.txt"},
|
||||||
|
},
|
||||||
|
Overwrite: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(manifest.ID)
|
||||||
|
fmt.Println(manifest.Files[0].Path)
|
||||||
|
// Output:
|
||||||
|
// reports.example
|
||||||
|
// report.txt
|
||||||
|
}
|
||||||
160
pkg/bundle/manifest.go
Normal file
160
pkg/bundle/manifest.go
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type rawManifest struct {
|
||||||
|
SchemaVersion *int `json:"schema_version"`
|
||||||
|
ID *string `json:"id"`
|
||||||
|
Digest *string `json:"digest"`
|
||||||
|
Created *string `json:"created"`
|
||||||
|
Files []rawManifestFile `json:"files"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawManifestFile struct {
|
||||||
|
Path *string `json:"path"`
|
||||||
|
SHA256 *string `json:"sha256"`
|
||||||
|
Size *int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseManifest(data []byte) (Manifest, error) {
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
var raw rawManifest
|
||||||
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("parse manifest: %w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := decoder.Decode(&extra); err != io.EOF {
|
||||||
|
return Manifest{}, fmt.Errorf("parse manifest: trailing data")
|
||||||
|
}
|
||||||
|
|
||||||
|
var manifest Manifest
|
||||||
|
if raw.SchemaVersion == nil {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest schema_version is required")
|
||||||
|
}
|
||||||
|
manifest.SchemaVersion = *raw.SchemaVersion
|
||||||
|
if raw.ID == nil || *raw.ID == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest id is required")
|
||||||
|
}
|
||||||
|
manifest.ID = *raw.ID
|
||||||
|
if raw.Digest == nil || *raw.Digest == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest digest is required")
|
||||||
|
}
|
||||||
|
manifest.Digest = *raw.Digest
|
||||||
|
if raw.Created == nil || *raw.Created == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest created is required")
|
||||||
|
}
|
||||||
|
created, err := time.Parse(time.RFC3339, *raw.Created)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest created must be RFC3339: %w", err)
|
||||||
|
}
|
||||||
|
manifest.Created = created
|
||||||
|
if len(raw.Files) == 0 {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest files is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
for index, rawFile := range raw.Files {
|
||||||
|
file, err := parseManifestFile(index, rawFile)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
manifest.Files = append(manifest.Files, file)
|
||||||
|
}
|
||||||
|
if err := ValidateManifest(manifest); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest %w", err)
|
||||||
|
}
|
||||||
|
return manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseManifestFile(index int, raw rawManifestFile) (ManifestFile, error) {
|
||||||
|
if raw.Path == nil || *raw.Path == "" {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].path is required", index)
|
||||||
|
}
|
||||||
|
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256 is required", index)
|
||||||
|
}
|
||||||
|
if raw.Size == nil {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].size is required", index)
|
||||||
|
}
|
||||||
|
return ManifestFile{
|
||||||
|
Path: *raw.Path,
|
||||||
|
SHA256: *raw.SHA256,
|
||||||
|
Size: *raw.Size,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func MarshalManifest(manifest Manifest) ([]byte, error) {
|
||||||
|
if err := ValidateManifest(manifest); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
data, err := json.MarshalIndent(manifest, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return append(data, '\n'), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func LoadManifest(root string) (Manifest, error) {
|
||||||
|
data, err := os.ReadFile(filepath.Join(root, ManifestName))
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("read manifest: %w", err)
|
||||||
|
}
|
||||||
|
return ParseManifest(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteManifest(root string, manifest Manifest, opts WriteManifestOptions) error {
|
||||||
|
data, err := MarshalManifest(manifest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
manifestPath := filepath.Join(root, ManifestName)
|
||||||
|
if !opts.Overwrite {
|
||||||
|
file, err := os.OpenFile(manifestPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("write manifest: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := file.Write(data); err != nil {
|
||||||
|
_ = file.Close()
|
||||||
|
return fmt.Errorf("write manifest: %w", err)
|
||||||
|
}
|
||||||
|
if err := file.Close(); err != nil {
|
||||||
|
return fmt.Errorf("write manifest: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := root
|
||||||
|
if dir == "" {
|
||||||
|
dir = "."
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(dir, ".manifest-*.tmp")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("write manifest temp: %w", err)
|
||||||
|
}
|
||||||
|
tmpPath := tmp.Name()
|
||||||
|
cleanup := true
|
||||||
|
defer func() {
|
||||||
|
if cleanup {
|
||||||
|
_ = os.Remove(tmpPath)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if _, err := tmp.Write(data); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return fmt.Errorf("write manifest temp: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return fmt.Errorf("write manifest temp: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmpPath, manifestPath); err != nil {
|
||||||
|
return fmt.Errorf("replace manifest: %w", err)
|
||||||
|
}
|
||||||
|
cleanup = false
|
||||||
|
return nil
|
||||||
|
}
|
||||||
31
pkg/bundle/path.go
Normal file
31
pkg/bundle/path.go
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const distributorStateName = ".distributor.json"
|
||||||
|
|
||||||
|
func ValidateSourcePath(value string) error {
|
||||||
|
if value == "" {
|
||||||
|
return fmt.Errorf("source path is required")
|
||||||
|
}
|
||||||
|
if strings.Contains(value, "\\") || strings.HasPrefix(value, "/") {
|
||||||
|
return fmt.Errorf("source path %q must be a clean relative slash-separated path", value)
|
||||||
|
}
|
||||||
|
if path.Clean(value) != value {
|
||||||
|
return fmt.Errorf("source path %q must be a clean relative slash-separated path", value)
|
||||||
|
}
|
||||||
|
for _, segment := range strings.Split(value, "/") {
|
||||||
|
if segment == "" || segment == "." || segment == ".." {
|
||||||
|
return fmt.Errorf("source path %q must be a clean relative slash-separated path", value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch value {
|
||||||
|
case ManifestName, distributorStateName:
|
||||||
|
return fmt.Errorf("%q is reserved", value)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
46
pkg/bundle/types.go
Normal file
46
pkg/bundle/types.go
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
const ManifestName = "manifest.json"
|
||||||
|
|
||||||
|
const SchemaVersion = 1
|
||||||
|
|
||||||
|
type Manifest struct {
|
||||||
|
SchemaVersion int `json:"schema_version"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
Created time.Time `json:"created"`
|
||||||
|
Files []ManifestFile `json:"files"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ManifestFile struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type BuildOptions struct {
|
||||||
|
Root string
|
||||||
|
ID string
|
||||||
|
Created time.Time
|
||||||
|
Files []string
|
||||||
|
Scan bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type WriteManifestOptions struct {
|
||||||
|
Overwrite bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type BundleFile struct {
|
||||||
|
SourcePath string
|
||||||
|
Path string
|
||||||
|
}
|
||||||
|
|
||||||
|
type WriteBundleOptions struct {
|
||||||
|
Root string
|
||||||
|
ID string
|
||||||
|
Created time.Time
|
||||||
|
Files []BundleFile
|
||||||
|
Overwrite bool
|
||||||
|
}
|
||||||
79
pkg/bundle/validate.go
Normal file
79
pkg/bundle/validate.go
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ValidateManifest(manifest Manifest) error {
|
||||||
|
if manifest.SchemaVersion != SchemaVersion {
|
||||||
|
return fmt.Errorf("schema_version must be %d", SchemaVersion)
|
||||||
|
}
|
||||||
|
if manifest.ID == "" {
|
||||||
|
return fmt.Errorf("id is required")
|
||||||
|
}
|
||||||
|
if err := ValidateDigest(manifest.Digest); err != nil {
|
||||||
|
return fmt.Errorf("digest: %w", err)
|
||||||
|
}
|
||||||
|
if manifest.Created.IsZero() {
|
||||||
|
return fmt.Errorf("created is required")
|
||||||
|
}
|
||||||
|
if len(manifest.Files) == 0 {
|
||||||
|
return fmt.Errorf("files is required")
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(manifest.Files))
|
||||||
|
for index, file := range manifest.Files {
|
||||||
|
if err := ValidateSourcePath(file.Path); err != nil {
|
||||||
|
return fmt.Errorf("files[%d].path: %w", index, err)
|
||||||
|
}
|
||||||
|
if err := ValidateDigest(file.SHA256); err != nil {
|
||||||
|
return fmt.Errorf("files[%d].sha256: %w", index, err)
|
||||||
|
}
|
||||||
|
if file.Size < 0 {
|
||||||
|
return fmt.Errorf("files[%d].size must be non-negative", index)
|
||||||
|
}
|
||||||
|
if _, exists := seen[file.Path]; exists {
|
||||||
|
return fmt.Errorf("files[%d].path duplicates %q", index, file.Path)
|
||||||
|
}
|
||||||
|
seen[file.Path] = struct{}{}
|
||||||
|
}
|
||||||
|
if actual := BundleDigest(manifest.Files); actual != manifest.Digest {
|
||||||
|
return fmt.Errorf("digest mismatch: got %s want %s", actual, manifest.Digest)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidateBundle(root string, manifest Manifest) error {
|
||||||
|
if err := ValidateManifest(manifest); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
files := append([]ManifestFile(nil), manifest.Files...)
|
||||||
|
for index, manifestFile := range files {
|
||||||
|
fullPath := filepath.Join(root, filepath.FromSlash(manifestFile.Path))
|
||||||
|
info, err := os.Lstat(fullPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("file %q stat: %w", manifestFile.Path, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("file %q must be a regular file", manifestFile.Path)
|
||||||
|
}
|
||||||
|
if info.Size() != manifestFile.Size {
|
||||||
|
return fmt.Errorf("file %q size mismatch: got %d want %d", manifestFile.Path, info.Size(), manifestFile.Size)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(fullPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("file %q read: %w", manifestFile.Path, err)
|
||||||
|
}
|
||||||
|
actualDigest := FileDigest(data)
|
||||||
|
if actualDigest != manifestFile.SHA256 {
|
||||||
|
return fmt.Errorf("file %q sha256 mismatch: got %s want %s", manifestFile.Path, actualDigest, manifestFile.SHA256)
|
||||||
|
}
|
||||||
|
files[index].SHA256 = actualDigest
|
||||||
|
files[index].Size = int64(len(data))
|
||||||
|
}
|
||||||
|
if actualDigest := BundleDigest(files); actualDigest != manifest.Digest {
|
||||||
|
return fmt.Errorf("digest mismatch: got %s want %s", actualDigest, manifest.Digest)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
187
pkg/bundle/writer.go
Normal file
187
pkg/bundle/writer.go
Normal file
@@ -0,0 +1,187 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
func WriteBundle(opts WriteBundleOptions) (Manifest, error) {
|
||||||
|
root, err := cleanBundleRoot(opts.Root)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
if opts.ID == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("id is required")
|
||||||
|
}
|
||||||
|
if len(opts.Files) == 0 {
|
||||||
|
return Manifest{}, fmt.Errorf("files is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
parent := filepath.Dir(root)
|
||||||
|
if err := os.MkdirAll(parent, 0o755); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("create bundle parent: %w", err)
|
||||||
|
}
|
||||||
|
if !opts.Overwrite {
|
||||||
|
if _, err := os.Lstat(root); err == nil {
|
||||||
|
return Manifest{}, fmt.Errorf("bundle root %q already exists", root)
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return Manifest{}, fmt.Errorf("stat bundle root: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tempRoot, err := os.MkdirTemp(parent, "."+filepath.Base(root)+"-*.tmp")
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("create bundle temp root: %w", err)
|
||||||
|
}
|
||||||
|
removeTemp := true
|
||||||
|
defer func() {
|
||||||
|
if removeTemp {
|
||||||
|
_ = os.RemoveAll(tempRoot)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
paths, err := copyBundleFiles(tempRoot, opts.Files)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
manifest, err := BuildManifest(BuildOptions{
|
||||||
|
Root: tempRoot,
|
||||||
|
ID: opts.ID,
|
||||||
|
Created: opts.Created,
|
||||||
|
Files: paths,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
if err := WriteManifest(tempRoot, manifest, WriteManifestOptions{}); err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
if err := ValidateBundle(tempRoot, manifest); err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := promoteBundleRoot(tempRoot, root, opts.Overwrite); err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
removeTemp = false
|
||||||
|
return manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanBundleRoot(root string) (string, error) {
|
||||||
|
if root == "" {
|
||||||
|
return "", fmt.Errorf("root is required")
|
||||||
|
}
|
||||||
|
return filepath.Clean(root), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyBundleFiles(root string, files []BundleFile) ([]string, error) {
|
||||||
|
paths := make([]string, 0, len(files))
|
||||||
|
seen := make(map[string]struct{}, len(files))
|
||||||
|
for index, file := range files {
|
||||||
|
if file.SourcePath == "" {
|
||||||
|
return nil, fmt.Errorf("files[%d].source_path is required", index)
|
||||||
|
}
|
||||||
|
if err := ValidateSourcePath(file.Path); err != nil {
|
||||||
|
return nil, fmt.Errorf("files[%d].path: %w", index, err)
|
||||||
|
}
|
||||||
|
if _, exists := seen[file.Path]; exists {
|
||||||
|
return nil, fmt.Errorf("files[%d].path duplicates %q", index, file.Path)
|
||||||
|
}
|
||||||
|
seen[file.Path] = struct{}{}
|
||||||
|
if err := copyBundleFile(root, file); err != nil {
|
||||||
|
return nil, fmt.Errorf("files[%d]: %w", index, err)
|
||||||
|
}
|
||||||
|
paths = append(paths, file.Path)
|
||||||
|
}
|
||||||
|
return paths, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyBundleFile(root string, file BundleFile) error {
|
||||||
|
info, err := os.Lstat(file.SourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("stat source %q: %w", file.SourcePath, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("source %q must be a regular file", file.SourcePath)
|
||||||
|
}
|
||||||
|
destination := filepath.Join(root, filepath.FromSlash(file.Path))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
|
||||||
|
return fmt.Errorf("create destination directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
source, err := os.Open(file.SourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("open source %q: %w", file.SourcePath, err)
|
||||||
|
}
|
||||||
|
defer source.Close()
|
||||||
|
|
||||||
|
mode := info.Mode().Perm()
|
||||||
|
if mode == 0 {
|
||||||
|
mode = 0o600
|
||||||
|
}
|
||||||
|
target, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create destination %q: %w", file.Path, err)
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(target, source); err != nil {
|
||||||
|
_ = target.Close()
|
||||||
|
return fmt.Errorf("copy to destination %q: %w", file.Path, err)
|
||||||
|
}
|
||||||
|
if err := target.Close(); err != nil {
|
||||||
|
return fmt.Errorf("close destination %q: %w", file.Path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func promoteBundleRoot(tempRoot, root string, overwrite bool) error {
|
||||||
|
if !overwrite {
|
||||||
|
if _, err := os.Lstat(root); err == nil {
|
||||||
|
return fmt.Errorf("bundle root %q already exists", root)
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("stat bundle root: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tempRoot, root); err != nil {
|
||||||
|
return fmt.Errorf("promote bundle root: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(root); errors.Is(err, os.ErrNotExist) {
|
||||||
|
if err := os.Rename(tempRoot, root); err != nil {
|
||||||
|
return fmt.Errorf("promote bundle root: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
} else if err != nil {
|
||||||
|
return fmt.Errorf("stat bundle root: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
backupRoot, err := reserveSiblingPath(filepath.Dir(root), "."+filepath.Base(root)+"-backup-*.tmp")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Rename(root, backupRoot); err != nil {
|
||||||
|
return fmt.Errorf("move existing bundle root: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tempRoot, root); err != nil {
|
||||||
|
restoreErr := os.Rename(backupRoot, root)
|
||||||
|
if restoreErr != nil {
|
||||||
|
return fmt.Errorf("promote bundle root: %w; restore existing bundle root: %v", err, restoreErr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("promote bundle root: %w", err)
|
||||||
|
}
|
||||||
|
_ = os.RemoveAll(backupRoot)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reserveSiblingPath(parent, pattern string) (string, error) {
|
||||||
|
path, err := os.MkdirTemp(parent, pattern)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("reserve backup path: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
return "", fmt.Errorf("reserve backup path: %w", err)
|
||||||
|
}
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
38
pkg/bundle/writer_integration_test.go
Normal file
38
pkg/bundle/writer_integration_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package bundle_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWriteBundleOutputValidatesThroughDistributor(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "bundle")
|
||||||
|
if err := os.WriteFile(filepath.Join(sourceRoot, "report.md"), []byte("# Report\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write source: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := bundle.WriteBundle(bundle.WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.distributor",
|
||||||
|
Files: []bundle.BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("WriteBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
if err := app.Validate(context.Background(), app.ValidateOptions{Path: outputRoot, Stdout: &stdout}); err != nil {
|
||||||
|
t.Fatalf("app.Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := stdout.String(), "Validated 1 bundle(s)\n"; got != want {
|
||||||
|
t.Fatalf("stdout = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
232
pkg/bundle/writer_test.go
Normal file
232
pkg/bundle/writer_test.go
Normal file
@@ -0,0 +1,232 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWriteBundleCreatesCompleteBundle(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "bundle")
|
||||||
|
writeFile(t, sourceRoot, "report.md", "# Report\n")
|
||||||
|
writeFile(t, sourceRoot, "summary.txt", "Summary\n")
|
||||||
|
created := time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
manifest, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.writer",
|
||||||
|
Created: created,
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "summary.txt"), Path: "nested/summary.txt"},
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !manifest.Created.Equal(created) {
|
||||||
|
t.Fatalf("created = %s, want %s", manifest.Created, created)
|
||||||
|
}
|
||||||
|
if paths := manifestPaths(manifest); !reflect.DeepEqual(paths, []string{"nested/summary.txt", "report.md"}) {
|
||||||
|
t.Fatalf("paths = %v, want caller order", paths)
|
||||||
|
}
|
||||||
|
if got := readFile(t, outputRoot, "nested/summary.txt"); got != "Summary\n" {
|
||||||
|
t.Fatalf("nested summary = %q", got)
|
||||||
|
}
|
||||||
|
if got := readFile(t, outputRoot, "report.md"); got != "# Report\n" {
|
||||||
|
t.Fatalf("report = %q", got)
|
||||||
|
}
|
||||||
|
loaded, err := LoadManifest(outputRoot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(loaded, manifest) {
|
||||||
|
t.Fatalf("loaded manifest = %#v, want %#v", loaded, manifest)
|
||||||
|
}
|
||||||
|
if err := ValidateBundle(outputRoot, loaded); err != nil {
|
||||||
|
t.Fatalf("ValidateBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteBundleDefaultsCreated(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "bundle")
|
||||||
|
writeFile(t, sourceRoot, "report.md", "# Report\n")
|
||||||
|
before := time.Now().UTC()
|
||||||
|
|
||||||
|
manifest, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.created",
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
after := time.Now().UTC()
|
||||||
|
if manifest.Created.Before(before) || manifest.Created.After(after) {
|
||||||
|
t.Fatalf("created = %s, want between %s and %s", manifest.Created, before, after)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteBundleRejectsDuplicatePaths(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
writeFile(t, sourceRoot, "report.md", "# Report\n")
|
||||||
|
|
||||||
|
_, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: filepath.Join(t.TempDir(), "bundle"),
|
||||||
|
ID: "reports.duplicate",
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "duplicates") {
|
||||||
|
t.Fatalf("WriteBundle() error = %v, want duplicate path error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteBundleRejectsSymlinkSource(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
writeFile(t, sourceRoot, "target.md", "# Report\n")
|
||||||
|
linkPath := filepath.Join(sourceRoot, "link.md")
|
||||||
|
if err := os.Symlink("target.md", linkPath); err != nil {
|
||||||
|
t.Skipf("symlink unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: filepath.Join(t.TempDir(), "bundle"),
|
||||||
|
ID: "reports.symlink",
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: linkPath, Path: "report.md"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "regular file") {
|
||||||
|
t.Fatalf("WriteBundle() error = %v, want regular file error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteBundleDoesNotOverwriteByDefault(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "bundle")
|
||||||
|
writeFile(t, sourceRoot, "report.md", "old\n")
|
||||||
|
if _, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.old",
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("initial WriteBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
writeFile(t, sourceRoot, "report.md", "new\n")
|
||||||
|
|
||||||
|
_, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.new",
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "already exists") {
|
||||||
|
t.Fatalf("WriteBundle() error = %v, want exists error", err)
|
||||||
|
}
|
||||||
|
if got := readFile(t, outputRoot, "report.md"); got != "old\n" {
|
||||||
|
t.Fatalf("report = %q, want old content", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteBundleOverwritesExistingRoot(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "bundle")
|
||||||
|
writeFile(t, sourceRoot, "old.md", "old\n")
|
||||||
|
if _, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.old",
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "old.md"), Path: "old.md"},
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("initial WriteBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
writeFile(t, sourceRoot, "new.md", "new\n")
|
||||||
|
|
||||||
|
manifest, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.new",
|
||||||
|
Overwrite: true,
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "new.md"), Path: "new.md"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteBundle(overwrite) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if manifest.ID != "reports.new" {
|
||||||
|
t.Fatalf("manifest id = %q, want reports.new", manifest.ID)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(outputRoot, "old.md")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("old file stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
if got := readFile(t, outputRoot, "new.md"); got != "new\n" {
|
||||||
|
t.Fatalf("new file = %q", got)
|
||||||
|
}
|
||||||
|
if err := ValidateBundle(outputRoot, manifest); err != nil {
|
||||||
|
t.Fatalf("ValidateBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteBundleKeepsExistingRootWhenReplacementBuildFails(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "bundle")
|
||||||
|
writeFile(t, sourceRoot, "report.md", "old\n")
|
||||||
|
oldManifest, err := WriteBundle(WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.old",
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "report.md"), Path: "report.md"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("initial WriteBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = WriteBundle(WriteBundleOptions{
|
||||||
|
Root: outputRoot,
|
||||||
|
ID: "reports.new",
|
||||||
|
Overwrite: true,
|
||||||
|
Files: []BundleFile{
|
||||||
|
{SourcePath: filepath.Join(sourceRoot, "missing.md"), Path: "report.md"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("WriteBundle(overwrite) error = nil, want missing source error")
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := LoadManifest(outputRoot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(loaded, oldManifest) {
|
||||||
|
t.Fatalf("loaded manifest = %#v, want old manifest %#v", loaded, oldManifest)
|
||||||
|
}
|
||||||
|
if got := readFile(t, outputRoot, "report.md"); got != "old\n" {
|
||||||
|
t.Fatalf("report = %q, want old content", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFile(t *testing.T, root, relative string) string {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(relative)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", relative, err)
|
||||||
|
}
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user