Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8366af6fb6 | |||
| 5e47d89355 | |||
| 2e0d903626 | |||
| 484fda2514 | |||
| 9a2eaf8e5e | |||
| 4e673dda76 | |||
| cf7e62733e | |||
| 91b72478d5 | |||
| 52078e2195 | |||
| 77cde40296 | |||
| 20129bfff5 | |||
| a95662226f | |||
| d23c624179 | |||
| 69043801d0 | |||
| 8b0ce4d134 | |||
| eba65018be | |||
| 11d1eabe2a | |||
| c02106987f | |||
| 598b665307 | |||
| c4e8ebff6f | |||
| 3da7f931b2 | |||
| b10a8bd194 | |||
| c84d8868d1 | |||
| fc33bbca54 |
10
docs/cli.md
10
docs/cli.md
@@ -72,7 +72,7 @@ distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
|
||||
|
||||
- `--config <path>` loads the pipeline configuration. If omitted, the application uses `/usr/local/etc/distributor/config.yml`.
|
||||
- `--dry-run` validates inputs and reports destination actions without applying changes.
|
||||
- `--force` permits a run when destination state indicates a conservative safety check would otherwise block it.
|
||||
- `--force` permits exceptional catalog replacement when a dry run reports `force_replace` for unmanaged content, a planned unmanaged path collision, invalid state, or unsupported future state.
|
||||
- `--format text|json` selects human-readable or machine-readable output.
|
||||
|
||||
`run` accepts no positional arguments.
|
||||
@@ -86,7 +86,7 @@ distributor reconcile-state --config <path> --pipeline <id> --destination <id> [
|
||||
- `--config <path>` loads the pipeline configuration and is required.
|
||||
- `--pipeline <id>` selects the pipeline used to identify the destination root and is required.
|
||||
- `--destination <id>` selects the destination root and is required.
|
||||
- `--all-owners` repairs missing managed output records for every owner in a shared-root state file. Without it, shared-root repair is scoped to the selected pipeline and destination owner.
|
||||
- `--all-owners` repairs missing managed output records for every owner in the selected catalog state file. Without it, repair is scoped to the selected pipeline and destination owner.
|
||||
- `--dry-run` reports repairs without rewriting `.distributor.json`.
|
||||
- `--format text|json` selects human-readable or machine-readable output.
|
||||
|
||||
@@ -206,7 +206,7 @@ go run ./cmd/distributor run --config examples/local-publish.yml --dry-run
|
||||
go run ./cmd/distributor run --config examples/local-publish.yml
|
||||
```
|
||||
|
||||
Use `--format json` when automation needs structured run results. Use `--force` only when the operator has reviewed the destination state conflict and intentionally wants to continue.
|
||||
Use `--format json` when automation needs structured run results. Use `--force` only after `--dry-run --force` reports the intended bounded `force_replace` action.
|
||||
|
||||
### Repair Destination State Records
|
||||
|
||||
@@ -229,7 +229,7 @@ go run ./cmd/distributor reconcile-state \
|
||||
--destination local-archive
|
||||
```
|
||||
|
||||
Use `--all-owners` only for shared-root destination state when all owners inside the selected root should be repaired.
|
||||
Use `--all-owners` only when every owner inside the selected catalog root should be repaired.
|
||||
|
||||
### Prune Managed Outputs
|
||||
|
||||
@@ -292,6 +292,8 @@ Text output is optimized for direct operator use. JSON output is optimized for a
|
||||
- Usage errors and fatal setup errors exit non-zero and do not emit a JSON result document.
|
||||
- `run --format json` emits a JSON result for partial destination failures, sets `ok` to `false`, includes result details and errors, and exits non-zero.
|
||||
- Warnings are included in JSON output and are printed in text output when relevant.
|
||||
- `run` summaries include `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and `fail_conflict` counters. Destination action records use the same stable action values and include the resolved `destination_path`.
|
||||
- `skip_same` means the planned outputs already match valid catalog metadata, so `run` does not write outputs, rewrite `.distributor.json`, delete files, or notify. The decision is based on catalog metadata and does not read destination file bytes.
|
||||
|
||||
## Diagnostics And Recovery
|
||||
|
||||
|
||||
209
docs/config.md
209
docs/config.md
@@ -2,11 +2,11 @@
|
||||
|
||||
Audience: administrators, operators, and advanced users who write YAML configuration for `distributor`.
|
||||
|
||||
This document is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operating procedures live in [Operations](operations.md), symptom-oriented recovery lives in [Troubleshooting](troubleshooting.md), and external contracts live under [Integrations](integrations/source-bundle.md).
|
||||
This is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operations guidance lives in [Operations](operations.md), recovery guidance lives in [Troubleshooting](troubleshooting.md), and file-format contracts live under [Integrations](integrations/source-bundle.md).
|
||||
|
||||
## Config File Loading
|
||||
|
||||
`distributor run --config <path>` and `distributor serve --config <path>` load the YAML file at `<path>`. If `--config` is omitted, both commands use:
|
||||
`distributor run --config <path>` and `distributor serve --config <path>` load the YAML file at `<path>`. If `--config` is omitted, commands use:
|
||||
|
||||
```text
|
||||
/usr/local/etc/distributor/config.yml
|
||||
@@ -21,7 +21,7 @@ Runtime backend support is command-specific:
|
||||
- `serve` uses `http_upload` sources through the HTTP upload API and publishes to configured `local`, `ssh`, and `s3` destinations.
|
||||
- `http_upload` is valid only as a source backend.
|
||||
|
||||
## Minimal Local Config
|
||||
## Minimal Working Config
|
||||
|
||||
```yaml
|
||||
pipelines:
|
||||
@@ -35,9 +35,9 @@ pipelines:
|
||||
path: /srv/reports/archive
|
||||
```
|
||||
|
||||
This config publishes source files only. It uses default validation, destination path mapping, publish, state, reconciliation, retention, transfer, and HTTP server values.
|
||||
This publishes source files only. It uses default validation, additive workflow, preserve-relative path mapping, source-only publish policy, disabled pruning, and default HTTP server values.
|
||||
|
||||
## Production-Oriented Local Config
|
||||
## Production-Oriented Config
|
||||
|
||||
```yaml
|
||||
server:
|
||||
@@ -61,23 +61,20 @@ pipelines:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /srv/reports/archive
|
||||
workflow: additive
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
path_mapping:
|
||||
mode: preserve_relative
|
||||
reconciliation:
|
||||
mode: replace
|
||||
transfer:
|
||||
on_destination_same: skip
|
||||
on_destination_older: replace
|
||||
on_destination_newer: skip
|
||||
on_conflict: fail
|
||||
retention:
|
||||
prune:
|
||||
enabled: false
|
||||
```
|
||||
|
||||
## HTTP Upload Source Config
|
||||
|
||||
HTTP upload sources are configured on pipelines and are served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML.
|
||||
HTTP upload sources are configured on pipelines and served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML.
|
||||
|
||||
```yaml
|
||||
server:
|
||||
@@ -97,15 +94,13 @@ pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
staging_path: /var/spool/distributor/weather-daily
|
||||
max_upload_size: 20MB
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /srv/reports/archive
|
||||
```
|
||||
|
||||
`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to. One token may authorize multiple upload pipelines, and multiple tokens may authorize the same upload pipeline.
|
||||
`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to.
|
||||
|
||||
For `http_upload` sources, `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`. `max_upload_size` defaults to `server.http.max_upload_size`.
|
||||
|
||||
@@ -136,7 +131,7 @@ See [Secrets](#secrets) for resolution rules.
|
||||
|
||||
Each token has:
|
||||
|
||||
- `id`: required unique slug-like identifier for the token record. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
|
||||
- `id`: required unique slug-like identifier for the token record.
|
||||
- `token_env`: required environment variable or secret-file name containing the bearer token value.
|
||||
- `allow_pipelines`: required non-empty list of configured pipeline ids whose source backend is `http_upload`.
|
||||
|
||||
@@ -148,12 +143,12 @@ Token values must resolve to non-empty strings and must be unique across token r
|
||||
|
||||
Each pipeline has:
|
||||
|
||||
- `id`: required unique slug-like identifier. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
|
||||
- `id`: required unique slug-like identifier.
|
||||
- `source`: required source backend config.
|
||||
- `validation`: optional validation policy.
|
||||
- `destinations`: required non-empty destination list.
|
||||
|
||||
Pipeline ids must be unique across the config.
|
||||
Slug-like identifiers must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
|
||||
|
||||
## Backend Reference
|
||||
|
||||
@@ -171,7 +166,7 @@ path: /srv/distributor/archive
|
||||
|
||||
### SSH/SFTP Backend
|
||||
|
||||
SSH backends use native SFTP and can be used as sources and destinations. Adapter protocol behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md).
|
||||
SSH backends use native SFTP and can be used as sources and destinations. Adapter behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md).
|
||||
|
||||
```yaml
|
||||
backend: ssh
|
||||
@@ -193,17 +188,11 @@ host_key_policy: strict
|
||||
- `known_hosts`: optional OpenSSH `known_hosts` path.
|
||||
- `host_key_policy`: optional host key policy. Default: `accept-new`.
|
||||
|
||||
Accepted host key policy values:
|
||||
|
||||
- `strict` or boolean `true`: require a matching known host key.
|
||||
- `accept-new`: accept and persist a new host key, but reject changed known keys.
|
||||
- `off` or boolean `false`: disable host key checking.
|
||||
|
||||
Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML.
|
||||
Accepted host key policy values are `strict` or boolean `true`, `accept-new`, and `off` or boolean `false`. Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML.
|
||||
|
||||
### S3-Compatible Backend
|
||||
|
||||
S3 backends can be used as sources and destinations. Adapter protocol behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md).
|
||||
S3 backends can be used as sources and destinations. Adapter behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md).
|
||||
|
||||
```yaml
|
||||
backend: s3
|
||||
@@ -255,42 +244,50 @@ Source bundle digest mismatches fail validation before destination writes occur.
|
||||
|
||||
## Destination Fields
|
||||
|
||||
Each destination embeds a backend config at the destination level and may also configure publishing, transforms, path mapping, links, state, reconciliation, retention, and transfer behavior.
|
||||
Each destination embeds a backend config at the destination level and may also configure workflow, publishing, transforms, path mapping, links, and retention.
|
||||
|
||||
```yaml
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /srv/reports/archive
|
||||
workflow: additive
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
path_mapping:
|
||||
mode: preserve_relative
|
||||
state:
|
||||
mode: single_owner
|
||||
reconciliation:
|
||||
mode: replace
|
||||
transfer:
|
||||
on_destination_same: skip
|
||||
on_destination_older: replace
|
||||
on_destination_newer: skip
|
||||
on_conflict: fail
|
||||
```
|
||||
|
||||
- `id`: required unique slug-like identifier within the pipeline.
|
||||
- Backend fields: required according to the selected destination backend.
|
||||
- `workflow`: optional catalog update workflow. Default: `additive`.
|
||||
- `publish`: optional publish policy. Default: source-only publication.
|
||||
- `transform`: required only when publishing generated HTML.
|
||||
- `path_mapping`: optional destination path mapping policy.
|
||||
- `links`: optional public URL metadata policy.
|
||||
- `state`: optional destination state ownership policy.
|
||||
- `reconciliation`: optional managed-output reconciliation policy.
|
||||
- `retention`: optional managed-output retention policy.
|
||||
- `transfer`: optional destination comparison action policy.
|
||||
|
||||
Destination ids must be unique within a pipeline.
|
||||
|
||||
Pre-workflow destination policy keys for state mode, conflict handling,
|
||||
ownership adoption, and per-comparison copy decisions are not accepted config
|
||||
fields. YAML files containing those keys fail during config loading.
|
||||
|
||||
## Destination Workflow
|
||||
|
||||
```yaml
|
||||
workflow: additive
|
||||
```
|
||||
|
||||
- `workflow`: optional. Accepted values are `additive` and `replacement`; default is `additive`.
|
||||
|
||||
`additive` writes planned outputs into the catalog and retains unrelated catalog-managed outputs in the same destination bundle path. Existing catalog records for planned paths are replaced by the current publication. A planned path that exists in storage but is not recorded in valid catalog state fails as unmanaged content unless `run --force` selects `force_replace`.
|
||||
|
||||
`replacement` writes planned outputs for the current pipeline and destination, and removes catalog outputs owned by the same pipeline and destination when those outputs are omitted from the new plan. Outputs owned by other pipeline/destination pairs remain catalog-managed. Replacement workflow is normal managed behavior and does not require `--force`.
|
||||
|
||||
Use `replacement` for stable latest-style destinations where the current owner should publish exactly the currently planned output set. Use `additive` when a destination root intentionally accumulates outputs over time or receives disjoint outputs from multiple configured destinations.
|
||||
|
||||
## Publish And Transform Policy
|
||||
|
||||
### Source-Only Publication
|
||||
@@ -313,6 +310,7 @@ transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
css_href: /assets/report.css
|
||||
```
|
||||
|
||||
`publish.html` controls whether generated HTML outputs are published. When `publish.html` is `true`, `transform.markdown_to_html.enabled` must also be `true`.
|
||||
@@ -322,10 +320,13 @@ Markdown transform fields:
|
||||
- `transform.markdown_to_html.enabled`: enables Markdown-to-HTML generation for this destination.
|
||||
- `transform.markdown_to_html.mode`: optional. Accepted values are `sidecar` and `index`; default is `sidecar` when a Markdown transform block is present.
|
||||
- `transform.markdown_to_html.input`: optional source manifest path for `index` mode only.
|
||||
- `transform.markdown_to_html.css_href`: optional stylesheet href to link from generated HTML.
|
||||
|
||||
`sidecar` mode renders every manifest-listed `.md` file to a same-directory `.html` output. `index` mode renders one Markdown source to `index.html` at the destination bundle path. If `index` mode omits `input`, the selected source bundle must contain exactly one Markdown file.
|
||||
|
||||
At least one output type must be enabled. Enabled Markdown transforms are rejected when `publish.html` is `false`, and `input` is rejected unless `mode` is `index`.
|
||||
`css_href` may be an absolute `http` or `https` URL, a root-relative path such as `/assets/report.css`, or a relative URL path such as `assets/report.css`. Query strings are allowed. `distributor` injects the href as a `<link rel="stylesheet">` element but does not copy, publish, verify, or manage the CSS file solely because `css_href` is set.
|
||||
|
||||
At least one output type must be enabled. Enabled Markdown transforms are rejected when `publish.html` is `false`, `input` is rejected unless `mode` is `index`, and `css_href` is rejected when the Markdown transform is disabled.
|
||||
|
||||
## Destination Path Mapping
|
||||
|
||||
@@ -361,93 +362,7 @@ Primary URL policies:
|
||||
- `html`: use the first generated HTML output.
|
||||
- `source`: use the first copied source output.
|
||||
|
||||
If no output matches the primary policy, per-output URLs may still be recorded and the top-level primary URL is omitted.
|
||||
|
||||
## Destination State Policy
|
||||
|
||||
```yaml
|
||||
state:
|
||||
mode: single_owner
|
||||
```
|
||||
|
||||
- `state.mode`: optional. Accepted values are `single_owner` and `shared_root`; default is `single_owner`.
|
||||
|
||||
`single_owner` state records one pipeline/destination owner for each destination bundle path and is the default state mode.
|
||||
|
||||
`shared_root` records multiple pipeline/destination owners in one destination root. Publish execution preserves unrelated owners, rejects path ownership conflicts, and writes shared-root destination state.
|
||||
|
||||
Use `shared_root` when multiple configured destinations intentionally write disjoint output paths into the same backend root:
|
||||
|
||||
```yaml
|
||||
pipelines:
|
||||
- id: reports-source
|
||||
source:
|
||||
backend: local
|
||||
path: /var/spool/distributor/reports
|
||||
destinations:
|
||||
- id: shared-root
|
||||
backend: local
|
||||
path: /srv/reports/shared
|
||||
state:
|
||||
mode: shared_root
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
- id: reports-html
|
||||
source:
|
||||
backend: local
|
||||
path: /var/spool/distributor/reports
|
||||
destinations:
|
||||
- id: shared-root
|
||||
backend: local
|
||||
path: /srv/reports/shared
|
||||
state:
|
||||
mode: shared_root
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
```
|
||||
|
||||
Every output path in a shared root belongs to exactly one `pipeline_id` and `destination_id`. A different owner planning the same path fails as a conflict.
|
||||
|
||||
## Reconciliation Policy
|
||||
|
||||
```yaml
|
||||
reconciliation:
|
||||
mode: replace
|
||||
```
|
||||
|
||||
- `reconciliation.mode`: optional. Accepted values are `replace` and `merge`; default is `replace`.
|
||||
|
||||
Reconciliation controls how a destination with older managed state is updated after transfer policy selects `replace_older`.
|
||||
|
||||
`replace` deletes the prior managed outputs recorded in `.distributor.json`, deletes the state file, verifies the destination bundle path is empty, then writes only the newly planned outputs and a new state file. This is the default and is appropriate when each publication should exactly match the current publish and transform policy.
|
||||
|
||||
`merge` keeps prior managed outputs that are not produced by the new plan. Planned paths already recorded in existing state may be overwritten; planned paths that already exist in storage but are not recorded as managed fail as unmanaged collisions. The resulting state file records the cumulative managed output set.
|
||||
|
||||
Example merge destination:
|
||||
|
||||
```yaml
|
||||
destinations:
|
||||
- id: static-site
|
||||
backend: local
|
||||
path: /srv/reports/site
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
reconciliation:
|
||||
mode: merge
|
||||
```
|
||||
|
||||
`links.primary_url` is selected from the newly planned outputs for the current run. Retained outputs keep their prior output metadata and timestamps.
|
||||
If no output matches the primary policy, per-output URLs may still be recorded and no primary URL is reported for the run.
|
||||
|
||||
## Retention Policy
|
||||
|
||||
@@ -465,28 +380,9 @@ retention:
|
||||
|
||||
When `retention.prune.enabled` is `true`, at least one of `older_than` or `keep_latest` is required. `older_than` must be greater than zero, and `keep_latest` must be zero or greater.
|
||||
|
||||
Pruning uses managed output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs.
|
||||
Pruning uses catalog output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs.
|
||||
|
||||
The `prune` command is owner-scoped for shared-root state. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`.
|
||||
|
||||
## Transfer Policy
|
||||
|
||||
```yaml
|
||||
transfer:
|
||||
on_destination_same: skip
|
||||
on_destination_older: replace
|
||||
on_destination_newer: skip
|
||||
on_conflict: fail
|
||||
```
|
||||
|
||||
Transfer fields and accepted values:
|
||||
|
||||
- `transfer.on_destination_same`: `skip` or `fail`. Default: `skip`.
|
||||
- `transfer.on_destination_older`: `replace` or `fail`. Default: `replace`.
|
||||
- `transfer.on_destination_newer`: `skip`, `replace`, or `fail`. Default: `skip`.
|
||||
- `transfer.on_conflict`: `fail` or `replace`. Default: `fail`.
|
||||
|
||||
`replace` for `on_destination_newer` and `on_conflict` is honored only when `run --force` is supplied. There is no config field that enables forced replacement by default.
|
||||
The `prune` command is scoped to the selected pipeline and destination owner. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`.
|
||||
|
||||
## Size And Duration Values
|
||||
|
||||
@@ -519,17 +415,12 @@ Defaults are applied after YAML decoding and before validation:
|
||||
- S3 `force_path_style: true`
|
||||
- `http_upload` source `staging_path: <server.http.staging_root>/<pipeline id>`
|
||||
- `http_upload` source `max_upload_size: server.http.max_upload_size`
|
||||
- `workflow: additive`
|
||||
- `publish.source: true` and `publish.html: false`
|
||||
- `transform.markdown_to_html.mode: sidecar` when a Markdown transform block is present and mode is omitted
|
||||
- `path_mapping.mode: preserve_relative`
|
||||
- `links.primary: auto` when a `links` block is present and `primary` is omitted
|
||||
- `state.mode: single_owner`
|
||||
- `reconciliation.mode: replace`
|
||||
- `retention.prune.enabled: false`
|
||||
- `transfer.on_destination_same: skip`
|
||||
- `transfer.on_destination_older: replace`
|
||||
- `transfer.on_destination_newer: skip`
|
||||
- `transfer.on_conflict: fail`
|
||||
|
||||
## Secrets
|
||||
|
||||
@@ -552,7 +443,7 @@ Fields resolved through this resolver:
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
Maintained examples live under [examples](../examples/). Config tests load every file listed here.
|
||||
Maintained examples live under [examples](../examples/). Config tests load these YAML files.
|
||||
|
||||
Local examples:
|
||||
|
||||
@@ -562,8 +453,8 @@ Local examples:
|
||||
- `local-index.yml`: local `index.html` publication.
|
||||
- `fan-out.yml`: local fan-out publication to source and HTML destinations.
|
||||
- `archive-and-latest.yml`: local archive plus fixed latest publication.
|
||||
- `merge-reconciliation.yml`: local HTML publication using merge reconciliation.
|
||||
- `shared-root.yml`: two local pipelines publishing disjoint outputs into one shared destination root.
|
||||
- `additive-workflow.yml`: two destinations publishing disjoint outputs into one catalog-managed root.
|
||||
- `replacement-workflow.yml`: fixed-path replacement workflow for a stable latest-style output set.
|
||||
- `http-upload-local.yml`: local HTTP upload server config; requires `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` in the process environment or as a secret-file name before running `serve`.
|
||||
|
||||
Environment-gated remote examples:
|
||||
|
||||
@@ -2,194 +2,37 @@
|
||||
|
||||
Audience: operators, integrators, and maintainers who inspect or reason about destination `.distributor.json` files.
|
||||
|
||||
Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for comparison, skip, replacement, and recovery decisions.
|
||||
Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for catalog planning, managed replacement, retention pruning, repair, and recovery.
|
||||
|
||||
## Single-Owner State Schema
|
||||
## Catalog State Schema
|
||||
|
||||
State written by `run` for `state.mode: single_owner` uses schema version `2`.
|
||||
Publish execution writes catalog state with `schema_version` `4`.
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 2,
|
||||
"distributor_version": "dev",
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"published_at": "2026-06-04T12:00:00Z",
|
||||
"created_at": "2026-06-04T12:00:00Z",
|
||||
"updated_at": "2026-06-04T12:00:00Z",
|
||||
"state": {
|
||||
"mode": "single_owner"
|
||||
},
|
||||
"reconciliation": {
|
||||
"mode": "replace"
|
||||
},
|
||||
"source": {
|
||||
"manifest": {
|
||||
"schema_version": 1,
|
||||
"id": "reports.example.2026-06-04",
|
||||
"digest": "sha256:...",
|
||||
"created": "2026-06-04T11:55:00Z",
|
||||
"files": [
|
||||
{"path": "report.md", "sha256": "sha256:...", "size": 1234}
|
||||
]
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"primary_url": "https://reports.example.com/archive/report.html"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"path": "report.html",
|
||||
"kind": "generated",
|
||||
"source_path": "report.md",
|
||||
"transform": "markdown_to_html",
|
||||
"url": "https://reports.example.com/archive/report.html",
|
||||
"sha256": "sha256:...",
|
||||
"size": 2345,
|
||||
"created_at": "2026-06-04T12:00:00Z",
|
||||
"updated_at": "2026-06-04T12:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Required fields:
|
||||
|
||||
- `schema_version`: must be `2` for newly written state.
|
||||
- `pipeline_id`: configured pipeline id that wrote the state.
|
||||
- `destination_id`: configured destination id that wrote the state.
|
||||
- `published_at`: RFC3339 publication timestamp.
|
||||
- `created_at`: RFC3339 timestamp for when this destination state record was first created.
|
||||
- `updated_at`: RFC3339 timestamp for the latest state update.
|
||||
- `state.mode`: must be `single_owner`.
|
||||
- `reconciliation.mode`: `replace` or `merge`.
|
||||
- `source.manifest`: embedded source bundle manifest.
|
||||
- `outputs`: output records array; empty is allowed, but the field is required.
|
||||
|
||||
Optional fields:
|
||||
|
||||
- `distributor_version`: application version string when available.
|
||||
- `links.primary_url`: absolute HTTP or HTTPS URL selected by destination link policy.
|
||||
|
||||
## Output Records
|
||||
|
||||
Each output record has:
|
||||
|
||||
- `path`: destination-relative output path.
|
||||
- `kind`: `source` or `generated`.
|
||||
- `source_path`: source manifest path used for the output.
|
||||
- `transform`: required for `generated` outputs; omitted for copied source outputs.
|
||||
- `url`: optional absolute HTTP or HTTPS URL for the output.
|
||||
- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes.
|
||||
- `size`: output byte size, zero or greater.
|
||||
- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed.
|
||||
- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state.
|
||||
|
||||
Output paths must be unique and use clean relative slash-separated path rules.
|
||||
|
||||
For replacement updates, newly planned outputs are written into state. For merge updates, retained output records preserve both timestamps, overwritten managed output records preserve `created_at` and receive a new `updated_at`, and new output records receive the current publication time for both fields.
|
||||
|
||||
## Reconciliation Semantics
|
||||
|
||||
Destination reconciliation applies when destination state is older than the source and transfer policy permits replacement.
|
||||
|
||||
- `replace`: for single-owner state, delete managed output paths recorded in `outputs` plus `.distributor.json`, require the destination bundle path to be empty afterward, write the newly planned outputs, and write state whose `outputs` are exactly that new planned set. For shared-root state, delete only the current owner's omitted outputs and preserve unrelated owners.
|
||||
- `merge`: retain managed output paths omitted from the new plan, overwrite planned paths only when they are already recorded in existing state, fail when a newly planned path exists in storage but is not recorded as managed, and write state whose `outputs` are the cumulative managed set.
|
||||
|
||||
Top-level `links.primary_url` is selected from the newly planned outputs for the current publication. Retained outputs keep their existing per-output URL metadata.
|
||||
|
||||
If a merge publication fails after writing outputs, cleanup removes only newly created outputs from that failed attempt. Previously managed overwritten paths remain managed and are not removed by failed-attempt cleanup.
|
||||
|
||||
## Comparison Semantics
|
||||
|
||||
`distributor` compares the current source manifest to destination state before writing:
|
||||
|
||||
- No state and no content: publish new outputs.
|
||||
- No state and existing content: treat the destination as unmanaged.
|
||||
- Shared-root state without the current owner: publish new outputs for that owner if planned paths do not collide with other owners or unmanaged content.
|
||||
- Matching embedded source manifest: skip.
|
||||
- Same source id with older `created`: replace if policy allows.
|
||||
- Same source id with newer `created`: skip by default.
|
||||
- Same source id and same `created` with different digest: conflict.
|
||||
- Different source id, pipeline id, or destination id: conflict.
|
||||
- Invalid state JSON or invalid state fields: conflict.
|
||||
|
||||
Normal single-owner replacement deletes only managed output paths recorded in `outputs` plus `.distributor.json`. Shared-root replacement deletes only omitted outputs for the current owner. Merge publication retains omitted managed outputs. Forced replacement deletes the bounded destination bundle path.
|
||||
|
||||
## State Repair Semantics
|
||||
|
||||
`distributor reconcile-state` can remove managed output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||
|
||||
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. The command checks output paths recorded in `outputs`, reports missing managed outputs, reports unmanaged entries under the destination root, and removes missing output records from valid state unless `--dry-run` is set.
|
||||
|
||||
For shared-root state, repair is scoped to the selected owner by default. With `--all-owners`, it checks and repairs missing output records for every owner in the selected shared-root state file.
|
||||
|
||||
State repair does not validate output digests, delete destination files, adopt unmanaged entries, or rewrite invalid or mismatched state.
|
||||
|
||||
## Prune Semantics
|
||||
|
||||
`distributor prune` can delete managed outputs selected by the configured destination retention policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||
|
||||
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
|
||||
|
||||
Prune planning uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`.
|
||||
|
||||
## Compatibility
|
||||
|
||||
`distributor` can read schema version `1` destination state for compatibility. When v1 state is read, it is treated as single-owner state with `reconciliation.mode: replace`. Missing top-level `created_at` and `updated_at` are inferred from `published_at`, and missing per-output timestamps are also inferred from `published_at`.
|
||||
|
||||
Newly written single-owner destination state from publish execution uses schema version `2`.
|
||||
|
||||
## Shared-Root State Schema
|
||||
|
||||
State written by `run` for `state.mode: shared_root` uses schema version `3`.
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 3,
|
||||
"schema_version": 4,
|
||||
"distributor_version": "dev",
|
||||
"created_at": "2026-06-04T12:00:00Z",
|
||||
"updated_at": "2026-06-04T12:10:00Z",
|
||||
"state": {
|
||||
"mode": "shared_root"
|
||||
"mode": "catalog"
|
||||
},
|
||||
"owners": [
|
||||
{
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"reconciliation": {
|
||||
"mode": "merge"
|
||||
},
|
||||
"source": {
|
||||
"manifest": {
|
||||
"schema_version": 1,
|
||||
"id": "reports.example.2026-06-04",
|
||||
"digest": "sha256:...",
|
||||
"created": "2026-06-04T11:55:00Z",
|
||||
"files": [
|
||||
{"path": "report.md", "sha256": "sha256:...", "size": 1234}
|
||||
]
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"primary_url": "https://reports.example.com/archive/report.html"
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"path": "report.html",
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "static-site",
|
||||
"source": {
|
||||
"id": "reports.example.2026-06-04",
|
||||
"digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"created": "2026-06-04T11:55:00Z"
|
||||
},
|
||||
"kind": "generated",
|
||||
"source_path": "report.md",
|
||||
"transform": "markdown_to_html",
|
||||
"url": "https://reports.example.com/archive/report.html",
|
||||
"sha256": "sha256:...",
|
||||
"sha256": "sha256:abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd",
|
||||
"size": 2345,
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"source_id": "reports.example.2026-06-04",
|
||||
"source_digest": "sha256:...",
|
||||
"source_created": "2026-06-04T11:55:00Z",
|
||||
"created_at": "2026-06-04T12:00:00Z",
|
||||
"updated_at": "2026-06-04T12:10:00Z"
|
||||
}
|
||||
@@ -197,55 +40,88 @@ State written by `run` for `state.mode: shared_root` uses schema version `3`.
|
||||
}
|
||||
```
|
||||
|
||||
Shared-root required fields:
|
||||
Top-level fields:
|
||||
|
||||
- `schema_version`: must be `3`.
|
||||
- `created_at`: RFC3339 timestamp for when this shared-root state record was first created.
|
||||
- `updated_at`: RFC3339 timestamp for the latest shared-root state update.
|
||||
- `state.mode`: must be `shared_root`.
|
||||
- `owners`: owner records keyed by `pipeline_id` and `destination_id`; each owner records its latest source manifest and reconciliation mode.
|
||||
- `outputs`: output records for every managed path under the shared destination root.
|
||||
- `schema_version`: required value `4`.
|
||||
- `distributor_version`: optional application version string.
|
||||
- `created_at`: RFC3339 timestamp for when this catalog record was first created.
|
||||
- `updated_at`: RFC3339 timestamp for the latest catalog update.
|
||||
- `state.mode`: required value `catalog`.
|
||||
- `outputs`: required array of catalog output records. Empty is valid.
|
||||
|
||||
Shared-root optional fields:
|
||||
## Output Records
|
||||
|
||||
- `distributor_version`: application version string when available.
|
||||
- `owners[].links.primary_url`: absolute HTTP or HTTPS URL selected by that owner's destination link policy.
|
||||
Each output record has:
|
||||
|
||||
Shared-root output records carry the same `path`, `kind`, `source_path`, `transform`, `url`, `sha256`, `size`, `created_at`, and `updated_at` fields as single-owner outputs. They also include the owner `pipeline_id` and `destination_id`, plus compact source identity fields `source_id`, `source_digest`, and `source_created`.
|
||||
- `path`: destination-bundle-relative output path.
|
||||
- `pipeline_id`: configured pipeline id that manages the output path.
|
||||
- `destination_id`: configured destination id that manages the output path.
|
||||
- `source`: compact source identity for the output.
|
||||
- `kind`: `source` or `generated`.
|
||||
- `source_path`: generated outputs only; source manifest path used to derive the output.
|
||||
- `transform`: generated outputs only; transform id, currently `markdown_to_html`.
|
||||
- `url`: optional absolute HTTP or HTTPS URL for the output.
|
||||
- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes.
|
||||
- `size`: output byte size, zero or greater.
|
||||
- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed.
|
||||
- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state.
|
||||
|
||||
## Shared-Root Ownership
|
||||
Output paths must be unique and use clean relative slash-separated path rules. `pipeline_id` and `destination_id` must be slug-like identifiers.
|
||||
|
||||
Shared-root state is owner-scoped by `pipeline_id` and `destination_id`.
|
||||
For copied source outputs, `source_path` and `transform` are omitted. For generated outputs, both fields are required.
|
||||
|
||||
- One output path may be owned by only one owner.
|
||||
- The same owner may overwrite its own managed paths.
|
||||
- A different owner planning an already owned path fails as a conflict.
|
||||
- A planned path that exists in storage but is not recorded in state fails as unmanaged content unless forced replacement is explicitly selected.
|
||||
## Source Identity
|
||||
|
||||
When an owner publishes, unrelated owner records and output records are preserved. The publishing owner's record is updated with the latest source manifest, reconciliation mode, and latest primary URL when present.
|
||||
Each output records the source identity that produced it:
|
||||
|
||||
## Shared-Root Timestamps
|
||||
- `source.id`: source manifest id.
|
||||
- `source.digest`: source manifest digest.
|
||||
- `source.created`: source manifest creation timestamp.
|
||||
|
||||
For shared-root state:
|
||||
The full source manifest is not embedded in catalog state. The source bundle's `manifest.json` remains the producer contract, and `.distributor.json` records only the source identity needed for catalog ownership and later maintenance workflows.
|
||||
|
||||
- top-level `created_at` remains the original shared-root state creation time;
|
||||
- top-level `updated_at` changes after a successful state write;
|
||||
- output `created_at` remains stable for an existing managed path;
|
||||
- output `updated_at` changes only when that path is rewritten;
|
||||
- newly managed output paths receive the publication time for both output timestamps.
|
||||
## Workflow Semantics
|
||||
|
||||
## Shared-Root Migration
|
||||
Destination `workflow` is runtime configuration and is not persisted in `.distributor.json`.
|
||||
|
||||
If `state.mode: shared_root` is configured and existing state is a compatible single-owner `.distributor.json` for the same pipeline id and destination id, the next successful publish writes schema version `3` shared-root state for that owner.
|
||||
`workflow: additive` writes planned outputs and retains unrelated catalog-managed outputs. If a planned path already has a catalog output record, the current publication replaces that record and overwrites the file. If a planned path exists in storage but is not recorded in valid catalog state, planning fails as unmanaged unless `run --force` selects `force_replace`.
|
||||
|
||||
If existing single-owner state belongs to a different pipeline or destination, publish fails as a conflict. `distributor` does not implicitly take over unrelated state or unmanaged files.
|
||||
`workflow: replacement` writes planned outputs for the current pipeline and destination and removes omitted outputs owned by that same pipeline and destination. Outputs owned by other pipeline/destination pairs remain catalog-managed. This is normal managed replacement and does not require `--force`.
|
||||
|
||||
`force_replace` is an explicit per-run recovery path. It deletes only the resolved destination bundle path, then writes planned outputs and fresh catalog state. It can replace unmanaged content, planned unmanaged path collisions, invalid destination state, and unsupported future destination state after dry-run review.
|
||||
|
||||
## Publish Planning Outcomes
|
||||
|
||||
Current run reports use these destination action labels:
|
||||
|
||||
- `publish_new`: no valid state exists and the destination bundle path is empty.
|
||||
- `upsert_additive`: valid catalog state exists and additive workflow will write the planned outputs.
|
||||
- `replace_catalog`: valid catalog state exists and replacement workflow will write the planned outputs and remove omitted outputs for the current owner.
|
||||
- `skip_same`: no-op action value in the run output vocabulary.
|
||||
- `force_replace`: explicit bounded destructive replacement selected by `--force`.
|
||||
- `fail_unmanaged`: unmanaged destination content prevents publication.
|
||||
- `fail_conflict`: invalid state or unsupported state prevents publication without explicit force.
|
||||
|
||||
Schema versions older than `4` are superseded legacy state for publish planning. Normal catalog planning may publish over superseded legacy state according to the configured workflow, while invalid state and unsupported future schema versions fail unless `--force` is explicitly selected.
|
||||
|
||||
## Repair Semantics
|
||||
|
||||
`distributor reconcile-state` removes catalog output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||
|
||||
By default, repair is scoped to output records whose `pipeline_id` and `destination_id` match the selected pipeline and destination. With `--all-owners`, it checks every catalog output record in the selected root.
|
||||
|
||||
The command reports missing managed outputs and unmanaged storage entries. Without `--dry-run`, it removes missing managed output records from valid catalog state and rewrites `.distributor.json`. It does not delete destination files, adopt unmanaged entries, validate output digests, or rewrite invalid state.
|
||||
|
||||
## Prune Semantics
|
||||
|
||||
`distributor prune` deletes catalog output paths selected by the configured destination `retention.prune` policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||
|
||||
Prune planning is scoped to output records whose `pipeline_id` and `destination_id` match the selected pipeline and destination. It uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid catalog state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`.
|
||||
|
||||
## Boundaries
|
||||
|
||||
Destination state is internal managed state written by `distributor`. Operators may inspect it during recovery, but normal workflows should not edit it by hand. Source `manifest.json` is not copied as destination state.
|
||||
|
||||
## Tests
|
||||
|
||||
Before changing this contract, inspect and run:
|
||||
|
||||
```sh
|
||||
|
||||
@@ -10,7 +10,7 @@ Rendering uses `github.com/yuin/goldmark`. The exact version is pinned in `go.mo
|
||||
|
||||
## Renderer Behavior
|
||||
|
||||
The transformer constructs `goldmark.New()` with no project-specific extensions, parser options, renderer options, templates, CSS, or metadata injection.
|
||||
The transformer constructs `goldmark.New()` with no project-specific extensions, parser options, renderer options, templates, or source manifest metadata injection.
|
||||
|
||||
Supported output modes:
|
||||
|
||||
@@ -19,6 +19,8 @@ Supported output modes:
|
||||
|
||||
In `index` mode, `transform.markdown_to_html.input` may name the source manifest path to render. If `input` is omitted, the source manifest must list exactly one `.md` file. The selected input must be a clean relative source path, must be listed in the source manifest, and must end in `.md`.
|
||||
|
||||
When `transform.markdown_to_html.css_href` is set, generated HTML includes a stylesheet link in the document head. The href may be an absolute HTTP(S) URL, a root-relative path, or a relative URL path. Distributor treats this as a link reference only; it does not copy, publish, verify, or manage the CSS file solely because `css_href` is configured.
|
||||
|
||||
Raw HTML embedded in Markdown is not passed through by the current renderer behavior. Tests allow Goldmark's disabled-or-escaped raw HTML output forms and reject literal script tags in generated HTML.
|
||||
|
||||
## HTML Wrapper
|
||||
@@ -28,10 +30,11 @@ Rendered Markdown body HTML is wrapped in a fixed document shell:
|
||||
- `<!doctype html>`
|
||||
- `<html lang="en">`
|
||||
- UTF-8 `<meta charset>`
|
||||
- optional `<link rel="stylesheet" href="...">` when `css_href` is configured
|
||||
- empty `<title>`
|
||||
- `<body>` containing the rendered Markdown body
|
||||
|
||||
The wrapper is deterministic and does not read configuration, templates, CSS, or source manifest metadata.
|
||||
The wrapper is deterministic. When `css_href` is omitted, the generated wrapper is unchanged from the unstyled output. When `css_href` is configured, its escaped link element is part of the generated output bytes.
|
||||
|
||||
## Output Metadata
|
||||
|
||||
@@ -47,7 +50,7 @@ Destination state stores generated outputs with `kind: generated`, `source_path`
|
||||
|
||||
## Boundaries
|
||||
|
||||
Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose transfer policy. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata.
|
||||
Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose catalog workflow behavior. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata.
|
||||
|
||||
## Tests
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ The adapter uses these S3 operations:
|
||||
|
||||
Writes buffer the input and set `ContentLength`. If no content type is supplied by the caller, the adapter infers a content type from the logical path.
|
||||
|
||||
Normal replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Merge publication retains omitted managed objects and may overwrite existing managed objects. Forced replacement deletes objects under the bounded destination bundle prefix. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
|
||||
Normal managed replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Forced replacement deletes objects under the bounded destination bundle prefix, then writes planned outputs and schema version `4` catalog state. For fixed-path destinations, that bounded prefix is the configured backend root. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
|
||||
|
||||
## Error Mapping
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ The configured `path` is the backend root. All source discovery, destination pat
|
||||
|
||||
The adapter rejects symlink ancestors for reads and writes. Reads require regular files. Writes create parent directories and prefer atomic temp-file-plus-rename writes when requested. Walk output is sorted through the shared storage walker.
|
||||
|
||||
Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Merge publication retains omitted managed paths and may overwrite existing managed paths. Forced replacement deletes the bounded destination bundle path.
|
||||
Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Forced replacement deletes the bounded destination bundle path, then writes planned outputs and schema version `4` catalog state. For fixed-path destinations, that bounded path is the configured backend root.
|
||||
|
||||
## Boundaries
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ User-facing command parsing stays in `internal/cli`, including `reconcile-state`
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, retention policy, and transfer policy.
|
||||
The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, workflow, publish policy, transform policy, path mapping, links, and retention policy.
|
||||
|
||||
Config fields are validated and defaulted by `internal/config` before app workflows use them.
|
||||
|
||||
@@ -32,11 +32,11 @@ The app layer registers default transforms, including Markdown-to-HTML, and supp
|
||||
|
||||
## State And Manifest Behavior
|
||||
|
||||
Run workflows discover and validate source bundles through `internal/bundle`. Destination state actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results.
|
||||
Run workflows discover and validate source bundles through `internal/bundle`. Destination catalog actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results. Run summaries count `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and `fail_conflict` separately.
|
||||
|
||||
Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing managed output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from state and rewrites valid state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid or mismatched state.
|
||||
Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing catalog output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from valid catalog state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid state.
|
||||
|
||||
Prune planning consumes a parsed destination state document and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state.
|
||||
Prune planning consumes parsed catalog state and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state.
|
||||
|
||||
Prune execution loads one configured pipeline/destination selector, opens that destination root, parses the root `.distributor.json`, and builds a plan from the destination retention policy. Dry-run returns the same planned and preserved managed output records without deleting files or rewriting state. Apply mode deletes only planned managed output paths, never unmanaged files or `.distributor.json`, then removes confirmed deleted records from state and updates the state timestamp. If a delete fails after earlier deletes succeeded, it rewrites state only for the confirmed deletions and preserves records for the failed and unattempted outputs so a retry remains accurate. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope.
|
||||
|
||||
@@ -54,9 +54,9 @@ HTTP upload status is in memory. Accepted jobs move through accepted, queued, ru
|
||||
|
||||
Runtime setup fails for config load, config validation, secret loading, or credential resolution errors. Source setup failures stop the affected run before destination planning. Destination open, planning, execution, and notification failures are recorded as destination failures where a partial result exists.
|
||||
|
||||
Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root all-owner repair still uses the selected destination to identify the root, then applies repair across owners inside that root. Invalid, unreadable, or ambiguous state fails before any rewrite.
|
||||
Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Catalog repair is scoped to that owner unless `--all-owners` is set. Invalid or unreadable state fails before any rewrite.
|
||||
|
||||
Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root pruning is scoped to the selected owner and preserves unrelated owners. Invalid, unreadable, or ambiguous state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output.
|
||||
Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Pruning is scoped to the selected owner and preserves unrelated owners. Invalid or unreadable state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output.
|
||||
|
||||
HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Upload requests can fail during authentication, idempotency-key validation, content-type validation, idempotency conflict checks, queue admission, archive staging, source validation, or later publish execution.
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ The canonical user-facing config reference is `docs/config.md`.
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, retention policy, and transfer policy.
|
||||
The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, destination workflow, publish policy, transform policy, path mapping, links, and retention policy.
|
||||
|
||||
## Adapters Used
|
||||
|
||||
@@ -26,11 +26,11 @@ No external storage adapters are used directly. The package exposes normalized c
|
||||
|
||||
## State And Manifest Behavior
|
||||
|
||||
The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as publish/transform combinations, links, state policy, reconciliation policy, retention policy, transfer policy, backend roots, S3 prefix shape, and HTTP upload source settings.
|
||||
The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as workflow, publish/transform combinations, links, retention policy, backend roots, S3 prefix shape, and HTTP upload source settings. Legacy destination policy YAML fields are rejected by strict decoding because they are not user-visible config fields.
|
||||
|
||||
## Skip And Resume Behavior
|
||||
|
||||
The package has no runtime skip or resume behavior. It provides transfer policy values that publish planning later applies to destination comparison outcomes.
|
||||
The package has no runtime skip or resume behavior. Publish planning later applies workflow values and per-run force options to destination catalog outcomes.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
|
||||
@@ -4,13 +4,13 @@ Audience: developers and LLM coding agents changing `internal/publish`.
|
||||
|
||||
## Purpose
|
||||
|
||||
`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns destination comparison mapping, output selection, URL planning, managed cleanup selection, replacement safety, and destination state projection.
|
||||
`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns output selection, URL planning, catalog action selection, managed cleanup selection, forced replacement safety, and destination state projection.
|
||||
|
||||
## Inputs And Outputs
|
||||
|
||||
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, state policy, reconciliation policy, transformer resolver, transfer policy, distributor version, and force flag.
|
||||
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, workflow, transformer resolver, distributor version, force flag, and request time.
|
||||
|
||||
Output from planning is a `Plan` with action, reason, destination identity, selected outputs, state mode, owner scope, reconciliation mode, optional existing single-owner or shared-root state, optional primary URL, and force metadata. Shared-root plans also expose other-owner outputs to preserve, current-owner outputs retained by merge, current-owner outputs deleted by replace, and current-owner outputs to write. Execution writes selected source outputs, generated outputs, and `.distributor.json` for executable publish or replacement actions.
|
||||
Output from planning is a `Plan` with action, reason, destination identity, selected outputs, workflow, owner scope, optional existing catalog state, optional superseded legacy marker, optional primary URL, catalog outputs to write, catalog outputs to retain, catalog outputs to delete, force metadata, and clear-root metadata. Execution writes selected source outputs, generated outputs, and schema version `4` `.distributor.json` for executable catalog actions.
|
||||
|
||||
## Boundaries
|
||||
|
||||
@@ -20,7 +20,7 @@ External destination state semantics are documented in `docs/integrations/destin
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
The package consumes already-defaulted config values for destination `publish`, `transform`, `links`, `state`, `reconciliation`, `transfer`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
|
||||
The package consumes already-defaulted config values for destination `workflow`, `publish`, `transform`, `links`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
|
||||
|
||||
## Adapters Used
|
||||
|
||||
@@ -28,27 +28,37 @@ The package depends on `internal/storage.Backend` for source and destination IO,
|
||||
|
||||
## State And Manifest Behavior
|
||||
|
||||
Planning inspects destination state through `internal/state`, compares it with the source manifest, and maps comparison outcomes plus transfer policy into actions: `publish_new`, `replace_older`, `force_replace`, `skip_same`, `skip_destination_newer`, `fail_conflict`, or `fail_unmanaged`.
|
||||
Planning inspects destination state through `internal/state` and maps catalog conditions into actions:
|
||||
|
||||
Single-owner destinations compare the whole destination state against the configured pipeline and destination ids. Shared-root destinations compare only the current owner scope, keyed by pipeline id and destination id. An absent shared-root owner is publishable for that owner unless a planned output collides with unmanaged storage content. Planned writes to a path owned by another shared-root owner fail as conflicts.
|
||||
- `publish_new`: no valid state exists and the destination bundle path is empty.
|
||||
- `upsert_additive`: additive workflow writes planned outputs and retains unrelated catalog outputs.
|
||||
- `replace_catalog`: replacement workflow writes planned outputs and deletes omitted outputs for the current owner.
|
||||
- `skip_same`: valid catalog metadata already matches every planned output.
|
||||
- `force_replace`: explicit bounded replacement selected by `Force`.
|
||||
- `fail_unmanaged`: unmanaged destination content blocks publication.
|
||||
- `fail_conflict`: invalid or unsupported state blocks publication.
|
||||
|
||||
Execution writes destination state after selected outputs are written. Destination state includes copied source output metadata, generated output metadata, output timestamps, embedded source manifest, reconciliation metadata, link metadata when configured, pipeline id, destination id, and publication timestamps.
|
||||
Superseded legacy state is identified by schema number and planned through catalog output projection. A successful publish writes schema version `4` catalog state.
|
||||
|
||||
## Skip And Resume Behavior
|
||||
Execution writes destination state after selected outputs are written. Catalog output records include owner identity, compact source identity, copied source output metadata, generated output metadata, output timestamps, and optional URL metadata.
|
||||
|
||||
`skip_same` and `skip_destination_newer` execute as no-ops. Replacement-mode single-owner updates remove managed output paths from existing state plus `.distributor.json`, verify the destination is empty, and write state whose outputs are exactly the new plan. Replacement-mode shared-root updates remove only current-owner omitted outputs and preserve unrelated owners. Merge-mode updates retain omitted managed outputs, overwrite only paths already recorded as managed, reject unmanaged destination path collisions, and write cumulative output state. Failed writes trigger cleanup where practical; merge cleanup removes only newly created outputs from the failed attempt.
|
||||
## Workflow Behavior
|
||||
|
||||
Shared-root execution writes schema version `3` state. It preserves unrelated owner records and outputs, updates only the publishing owner metadata, preserves root `created_at`, and updates root `updated_at` after successful state writes. Compatible single-owner state for the same pipeline and destination is converted to shared-root state on successful publish.
|
||||
Additive workflow computes a write set for the planned outputs and preserves catalog outputs for unplanned paths. Existing catalog records for planned paths are replaced by the current owner and source identity. When every planned output already matches catalog metadata, planning returns `skip_same` and ignores unrelated retained catalog outputs for the no-op decision.
|
||||
|
||||
Forced replacement is explicit per request and deletes the bounded destination bundle path before writing new outputs and state.
|
||||
Replacement workflow computes a write set for the planned outputs, preserves other-owner outputs, and deletes omitted outputs owned by the current pipeline and destination. It does not need `Force`. Matching planned outputs return `skip_same` only when replacement would not delete omitted outputs for the current owner.
|
||||
|
||||
Catalog skip comparison is metadata-only. It checks pipeline id, destination id, source id, source digest, source creation timestamp, output path, kind, digest, size, generated output source path, generated output transform, and output URL metadata. It does not read destination file bytes, and `skip_same` execution does not write outputs, rewrite `.distributor.json`, delete files, or notify.
|
||||
|
||||
Forced replacement is explicit per request. It deletes the bounded destination bundle path before writing planned outputs and schema version `4` catalog state. Catalog planning selects `force_replace` only when `Force` is true and normal planning would otherwise fail for a non-empty no-state destination, a planned path collision with unmanaged storage content, invalid destination state, or unsupported future destination state.
|
||||
|
||||
Retention pruning is not part of publish execution and does not run automatically after a successful publish. The app-level prune workflow uses destination state after publication to select managed outputs for deletion.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Planning fails for incomplete requests, invalid publish/transform policy, invalid state mode, invalid reconciliation mode, output path collisions, invalid destination state, unmanaged destination content without force, shared-root owner path conflicts, conflict outcomes not allowed by transfer policy, unresolved transforms, invalid Markdown output selection, and invalid link URL planning.
|
||||
Planning fails for incomplete requests, invalid publish/transform policy, invalid workflow, output path collisions, unresolved transforms, invalid Markdown output selection, invalid link URL planning, invalid destination state without force, unmanaged destination content without force, and unsupported future state without force.
|
||||
|
||||
Execution fails on delete, read, transform output, unmanaged merge path collision, shared-root ownership conflict, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable publish or replacement actions.
|
||||
Execution fails on delete, read, transform output, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable catalog publish or replacement actions.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
@@ -62,12 +72,9 @@ Execution fails on delete, read, transform output, unmanaged merge path collisio
|
||||
- Planning is deterministic for the same request and destination state.
|
||||
- Destination bundle paths are caller-supplied and backend-root-relative.
|
||||
- URL generation uses URL path semantics and never infers public URLs from backend config.
|
||||
- Replacement reconciliation deletes only managed paths recorded in existing state plus `.distributor.json` for single-owner state, and only current-owner omitted outputs for shared-root state.
|
||||
- Merge reconciliation never adopts unmanaged content.
|
||||
- Merge state output records are cumulative for the single owner.
|
||||
- Shared-root planning is owner-scoped and preserves unrelated owner outputs.
|
||||
- Shared-root execution writes owner-scoped changes without deleting unrelated owners.
|
||||
- Forced replacement deletes only within the supplied destination bundle path.
|
||||
- Additive workflow never adopts unmanaged storage content.
|
||||
- Replacement workflow deletes only catalog outputs owned by the current pipeline and destination unless force replacement clears the bounded destination bundle path.
|
||||
- Forced replacement deletes only within the supplied destination bundle path and then writes catalog state using the same output projection as normal catalog publish planning.
|
||||
- Destination state is written after selected outputs are written.
|
||||
- Transform resolution stays behind a caller-supplied interface.
|
||||
- Unmanaged content is claimed only by explicit force.
|
||||
|
||||
@@ -4,21 +4,21 @@ Audience: developers and LLM coding agents changing `internal/state`.
|
||||
|
||||
## Purpose
|
||||
|
||||
`internal/state` parses, validates, serializes, and compares `.distributor.json` destination state.
|
||||
`internal/state` parses, validates, serializes, and updates `.distributor.json` destination catalog state records.
|
||||
|
||||
## Inputs And Outputs
|
||||
|
||||
Inputs are destination state JSON, constructed state values, current source manifest, pipeline id, destination id, and whether the destination path has content without state. Outputs are validated state values, JSON bytes, comparison outcomes, and human-readable reasons.
|
||||
Inputs are destination state JSON, constructed catalog values, owner scopes, managed output paths, timestamps, and prune policy inputs. Outputs are validated catalog values, JSON bytes, managed path lists, owner-filtered output lists, missing-output repair results, and prune candidate plans.
|
||||
|
||||
## Boundaries
|
||||
|
||||
The package does not inspect storage backends, mutate files, choose transfer policy, build publish outputs, generate URLs, or parse config. Publish planning consumes state comparison outcomes.
|
||||
The package does not inspect storage backends, mutate files, choose workflow actions, build publish outputs, generate URLs, or parse config. Publish planning consumes parsed catalog state and catalog output helpers.
|
||||
|
||||
The external destination state contract is documented in `docs/integrations/destination-state.md`.
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
`internal/state` uses config state mode and reconciliation mode constants for destination state validation and legacy state normalization. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
|
||||
`internal/state` uses shared constants for catalog mode, output kinds, slug-like id validation, link validation, storage path validation, and source manifest validation. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
|
||||
|
||||
## Adapters Used
|
||||
|
||||
@@ -26,42 +26,34 @@ None.
|
||||
|
||||
## State And Manifest Behavior
|
||||
|
||||
`.distributor.json` schema version is `2` for newly written single-owner state. Required fields are `pipeline_id`, `destination_id`, `published_at`, `created_at`, `updated_at`, `state.mode`, `reconciliation.mode`, `source.manifest`, and `outputs`. `distributor_version` and `links` are optional.
|
||||
Current `.distributor.json` publish output uses schema version `4` catalog state. Required top-level fields are `schema_version`, `created_at`, `updated_at`, `state.mode`, and `outputs`; `distributor_version` is optional.
|
||||
|
||||
Schema version `1` state remains readable. Parsing infers `state.mode: single_owner`, `reconciliation.mode: replace`, top-level `created_at` and `updated_at` from `published_at`, and per-output timestamps from `published_at`.
|
||||
Each catalog output record requires a clean path, pipeline id, destination id, source identity, `source` or `generated` kind, lowercase SHA-256 digest, non-negative size, and created/updated timestamps. Generated outputs require `source_path` and `transform`; copied source outputs must omit both. Stored URLs are optional and must pass `internal/link` validation.
|
||||
|
||||
Schema version `3` is shared-root state. It records `state.mode: shared_root`, shared state timestamps, owner records keyed by pipeline id and destination id, each owner's latest source manifest and reconciliation metadata, optional owner primary links, and output records for every managed path. Shared-root output records include owner ids and compact source identity fields for source id, digest, and creation time.
|
||||
Embedded source identity records contain source manifest id, digest, and creation timestamp. Full source manifests are not embedded in catalog state.
|
||||
|
||||
Shared-root publish conversion is explicit. Compatible single-owner state for the same pipeline and destination can be projected into the current owner scope by publish execution. Unrelated single-owner state remains a conflict.
|
||||
The package identifies schema versions older than the current catalog schema as superseded legacy state for publish planning. It rejects invalid JSON, malformed catalog state, and unsupported future schema versions.
|
||||
|
||||
Embedded source manifests are parsed and validated through `internal/bundle`, which delegates source manifest semantics to `pkg/bundle`. Output records require clean paths, `source` or `generated` kind, valid source paths, lowercase SHA-256 digests, non-negative sizes, created and updated timestamps, and transform ids for generated outputs. Stored URLs must pass `internal/link` validation.
|
||||
The package provides helpers for finding catalog outputs by path, filtering outputs by owner, listing managed output paths, removing missing output records for one owner or every owner, and building owner-scoped prune candidates.
|
||||
|
||||
The package also provides helpers for finding output records by path, projecting planned publish outputs into timestamped state outputs, merging retained and newly planned output records, computing managed output paths from single-owner state, removing missing managed output records from single-owner state, and building prune candidates from managed outputs.
|
||||
|
||||
For shared-root state, helpers parse either state shape, identify the current owner scope, return an owner's latest source manifest, list managed paths for one owner or all owners, detect path ownership conflicts, project planned owner outputs, merge one owner's planned outputs while preserving unrelated owners, replace one owner's outputs by removing that owner's omitted outputs, remove missing managed output records for either the current owner or every owner, and build owner-scoped prune candidates.
|
||||
|
||||
Shared-root helper projections preserve output `created_at` for existing managed paths and use the current publication time for rewritten `updated_at`. Root-level `created_at` preservation is owned by publish execution.
|
||||
Publish execution owns catalog output projection and timestamp preservation for rewritten outputs. State helpers only parse, validate, filter, and remove catalog records supplied by callers.
|
||||
|
||||
## Skip And Resume Behavior
|
||||
|
||||
Comparison is pure. It returns outcomes for absent state, unmanaged content, invalid state, pipeline/destination mismatch, same source manifest, older destination, newer destination, same-created digest conflict, and different source id conflict. It does not decide whether to skip, replace, force, or fail; publish planning maps outcomes to actions.
|
||||
Catalog parsing and helper transformations are pure. State code does not decide whether to skip, upsert, replace, force, or fail; publish planning maps parsed state and storage observations to actions.
|
||||
|
||||
Shared-root comparison is owner-scoped. It compares only the owner keyed by the current pipeline id and destination id, treats an absent owner as absent destination state for that owner, and can compare compatible single-owner state for the current owner without converting unrelated single-owner state.
|
||||
Missing-output removal helpers remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers.
|
||||
|
||||
Missing-output removal helpers are pure state transformations used by app-level state repair. They remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers.
|
||||
|
||||
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution and the `prune` command use the missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
|
||||
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution uses missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid state mode, invalid reconciliation mode, invalid embedded manifests, duplicate owners, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transforms, invalid URLs, invalid digests, negative sizes, and shared-root outputs whose owner is not registered.
|
||||
Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid catalog mode, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transform metadata, invalid URLs, invalid digests, and negative sizes.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/state/distributor_test.go`
|
||||
- `internal/state/shared_root_test.go`
|
||||
- `internal/state/catalog_test.go`
|
||||
- `internal/state/prune_test.go`
|
||||
- `internal/state/compare_test.go`
|
||||
- `internal/app/reconcile_state_test.go`
|
||||
- `internal/cli/reconcile_state_test.go`
|
||||
- `internal/publish/*_test.go`
|
||||
@@ -69,15 +61,13 @@ Parsing rejects invalid JSON, trailing data, missing required fields, invalid ti
|
||||
## Architectural Invariants
|
||||
|
||||
- `.distributor.json` is the destination sentinel and state record.
|
||||
- Comparison does not mutate storage.
|
||||
- Embedded source manifests use the source bundle contract.
|
||||
- Newly written single-owner state uses schema version `2`.
|
||||
- Schema version `1` state remains readable as replacement-mode single-owner state.
|
||||
- Schema version `3` shared-root state is parsed and validated without converting unrelated single-owner state.
|
||||
- Shared-root owner updates preserve unrelated owners and reject planned path collisions with other owners.
|
||||
- State helpers do not inspect or mutate storage.
|
||||
- Source identity uses the source bundle contract.
|
||||
- Newly written publish state uses schema version `4`.
|
||||
- Superseded legacy schema handling is limited to identifying older state for publish planning.
|
||||
- Missing-output repair helpers preserve unrelated owner records and outputs.
|
||||
- Prune planning uses output `updated_at` and preserves unrelated shared-root owners.
|
||||
- Generated outputs always record a transform id.
|
||||
- Prune planning uses output `updated_at` and preserves unrelated owners.
|
||||
- Generated outputs always record a transform id and source path.
|
||||
- Output records always carry created and updated timestamps after parsing.
|
||||
- Stored URLs are optional and must be absolute HTTP or HTTPS URLs when present.
|
||||
- `distributor_version` is diagnostic metadata, not a comparison key.
|
||||
|
||||
@@ -56,40 +56,33 @@ Destination path mapping controls where each source bundle is published beneath
|
||||
|
||||
Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple bundles have the same timestamp, the source-root-relative bundle path in ascending order wins.
|
||||
|
||||
Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination state record. It records the pipeline id, destination id, publication time, state mode, reconciliation mode, source manifest, copied outputs, generated outputs, output timestamps, and optional public URL metadata.
|
||||
Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination catalog. It records catalog schema version `4`, output owners, source identity for each output, output digests and sizes, timestamps, and optional URL metadata.
|
||||
|
||||
`manifest.json` from the source bundle is not copied as destination state.
|
||||
|
||||
## Destination State And Retry Behavior
|
||||
## Catalog Publish Behavior
|
||||
|
||||
`distributor` compares the source manifest to destination `.distributor.json` before writing:
|
||||
`distributor` plans from the current source bundle, destination workflow, destination storage content, and `.distributor.json`.
|
||||
|
||||
- No destination state and no destination content: publish new outputs.
|
||||
- Matching destination state: skip as already published.
|
||||
- Older destination state for the same source id: replace if transfer policy allows it.
|
||||
- Newer destination state: skip by default.
|
||||
- Invalid destination state, identity mismatch, different source id, or same-created digest mismatch: fail by default.
|
||||
- Content without `.distributor.json`: fail as unmanaged content by default.
|
||||
- No valid state and no destination content: publish new outputs.
|
||||
- No valid state and existing destination content: fail as unmanaged unless `--force` is used.
|
||||
- Valid catalog state with `workflow: additive`: write planned outputs and retain unrelated catalog outputs.
|
||||
- Valid catalog state with `workflow: replacement`: write planned outputs and remove omitted outputs owned by the selected pipeline and destination.
|
||||
- Planned output path exists in storage but is not recorded in valid catalog state - fail as unmanaged unless `--force` is used.
|
||||
- Invalid destination state or unsupported future state - fail as conflict unless `--force` is used.
|
||||
- Superseded legacy state - publish through the catalog planner and write schema version `4` state on success.
|
||||
|
||||
When destination state is older than the source, `transfer.on_destination_older` controls whether publication may proceed and `reconciliation.mode` controls how managed outputs are updated.
|
||||
`workflow: additive` is the default. It is useful for archive roots, fan-out roots that intentionally receive disjoint outputs, and roots that accumulate managed outputs over time.
|
||||
|
||||
`reconciliation.mode: replace` is the default. It deletes only managed output paths recorded in `.distributor.json` plus the state file, verifies the destination bundle path is empty, then writes the newly planned outputs and state. The new state `outputs` array is exactly the newly planned output set.
|
||||
|
||||
`reconciliation.mode: merge` retains prior managed outputs that are omitted from the new plan. It overwrites planned paths only when those paths are already recorded in existing state as managed. If a newly planned path already exists in storage but is not recorded in state, publication fails as an unmanaged path collision. The new state `outputs` array is the cumulative managed output set.
|
||||
|
||||
For both modes, retained or overwritten paths are identified only from `.distributor.json`; unmanaged files are not adopted.
|
||||
|
||||
For `state.mode: shared_root`, one destination root may contain outputs from multiple pipeline/destination owners. Comparisons, replacement, and merge retention are scoped to the current owner. Outputs owned by other owners are preserved. A planned output path owned by another owner fails as a conflict, and a planned path that exists in storage but is not recorded in state fails as unmanaged content by default.
|
||||
|
||||
If `state.mode: shared_root` is configured on a destination whose existing single-owner state belongs to the same pipeline and destination, the next successful publish converts that state file to shared-root schema. Existing single-owner state for a different pipeline or destination remains a conflict.
|
||||
|
||||
If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. In `replace` mode, cleanup removes outputs written during that failed attempt. In `merge` mode, cleanup removes only newly created outputs from that failed attempt; overwritten managed outputs are left in place because they previously belonged to the managed set. Operators should still inspect the destination after a failed write before retrying.
|
||||
`workflow: replacement` is useful for stable latest-style roots where the current pipeline and destination should leave only the currently planned output set for that owner. Replacement workflow is normal managed behavior and does not require `--force`.
|
||||
|
||||
Fan-out destinations are independent. If one destination fails after planning or execution begins, later destinations are still attempted. The command exits non-zero if any destination failed.
|
||||
|
||||
If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. Operators should still inspect the destination bundle path after a failed write before retrying.
|
||||
|
||||
## Destination State Repair
|
||||
|
||||
Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the state record only; it does not restore missing files.
|
||||
Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the catalog record only; it does not restore missing files.
|
||||
|
||||
Preview the repair first:
|
||||
|
||||
@@ -110,11 +103,9 @@ go run ./cmd/distributor reconcile-state \
|
||||
--destination <destination-id>
|
||||
```
|
||||
|
||||
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks each managed output path with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting.
|
||||
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks matching catalog output paths with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting.
|
||||
|
||||
Without `--dry-run`, it removes missing managed output records from valid state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid or mismatched state.
|
||||
|
||||
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, repair is scoped to the selected owner by default. Add `--all-owners` only when every owner in the selected shared-root state should have missing managed output records removed.
|
||||
Without `--dry-run`, it removes missing managed output records from valid catalog state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid state. Add `--all-owners` only when every catalog owner inside the selected root should be repaired.
|
||||
|
||||
## Managed Output Pruning
|
||||
|
||||
@@ -142,28 +133,28 @@ go run ./cmd/distributor prune \
|
||||
|
||||
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, and plans from the selected destination's `retention.prune` policy. It uses managed output `updated_at` timestamps. When both `keep_latest` and `older_than` are configured, it preserves the newest `keep_latest` outputs before applying the age policy.
|
||||
|
||||
`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate.
|
||||
|
||||
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
|
||||
`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths for the selected pipeline/destination owner, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate.
|
||||
|
||||
## Dry Runs And Output Review
|
||||
|
||||
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, or SSH `known_hosts` entries. For reconciliation, dry runs report the same high-level action labels as execution; inspect the configured destination's `reconciliation.mode` to determine whether `replace_older` will replace the managed set or merge into it.
|
||||
|
||||
For shared-root destinations, dry runs are owner-scoped. A `replace_older` action replaces or merges only the current owner according to `reconciliation.mode`; unrelated owners remain managed by the shared-root state.
|
||||
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, notifier events, or SSH `known_hosts` entries.
|
||||
|
||||
Review these action labels before publishing:
|
||||
|
||||
- `publish_new`: destination state is absent, or a shared-root owner is absent and planned paths are publishable.
|
||||
- `replace_older`: destination state is older than the source.
|
||||
- `skip_same`: destination state already matches the source.
|
||||
- `skip_destination_newer`: destination state is newer than the source and is skipped.
|
||||
- `force_replace`: destructive replacement selected because `--force` is present and policy permits it.
|
||||
- `error`: planning or execution failed for that destination.
|
||||
- `publish_new`: destination state is absent and the destination bundle path is empty.
|
||||
- `upsert_additive`: additive workflow will write planned outputs into valid catalog state.
|
||||
- `replace_catalog`: replacement workflow will write planned outputs and remove omitted outputs for the current owner.
|
||||
- `skip_same`: planned outputs already match valid catalog metadata, so publication will not write outputs, rewrite `.distributor.json`, delete files, or notify.
|
||||
- `force_replace`: destructive catalog replacement selected because `--force` is present for unmanaged content, a planned unmanaged path collision, invalid state, or unsupported future state.
|
||||
- `fail_unmanaged`: unmanaged destination content prevents publication.
|
||||
- `fail_conflict`: invalid or unsupported destination state prevents publication.
|
||||
- `error`: setup, planning, or execution failed for that destination.
|
||||
|
||||
Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced.
|
||||
Text and JSON summaries count `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, `fail_conflict`, and failed destinations separately. JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document.
|
||||
|
||||
JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document.
|
||||
The `skip_same` optimization trusts valid catalog metadata. It compares owner identity, source identity, output path, kind, digest, size, generated output metadata, and URL metadata recorded in `.distributor.json`; it does not read destination file bytes to detect bitrot.
|
||||
|
||||
Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced. For fixed destinations, the resolved destination bundle path is the backend root.
|
||||
|
||||
## Forced Replacement Workflow
|
||||
|
||||
@@ -174,14 +165,9 @@ go run ./cmd/distributor run --config <config-path> --dry-run --force
|
||||
go run ./cmd/distributor run --config <config-path> --force
|
||||
```
|
||||
|
||||
Forced replacement can claim unmanaged non-empty destination paths. State conflicts require both `--force` and transfer policy that permits replacement:
|
||||
Forced replacement can claim a non-empty destination path with no valid `.distributor.json`, replace planned output paths that collide with storage content not recorded in valid catalog state, and recover from invalid or unsupported future destination state. It is reserved for exceptional destructive replacement. Valid catalog-managed additive upserts and replacement workflow publishes do not require `--force`.
|
||||
|
||||
- newer destination state requires `transfer.on_destination_newer: replace`;
|
||||
- conflict outcomes require `transfer.on_conflict: replace`.
|
||||
|
||||
Forced replacement deletes the current destination bundle path before writing outputs and state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the backend root, so a forced replacement can clear that configured root.
|
||||
|
||||
For shared-root destinations, forced replacement also deletes the configured destination bundle path before writing new shared-root state. This removes unrelated owners inside that destination root. Preview with `--dry-run --force` and confirm the destination path before applying.
|
||||
Forced replacement deletes the current destination bundle path before writing planned outputs and schema version `4` catalog state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the configured backend root, so a forced replacement can clear that configured root. Dry-run text and JSON output report that root as `target=.` or `destination_path: "."`.
|
||||
|
||||
`--force` applies only to the current invocation. There is no config field that enables forced replacement by default.
|
||||
|
||||
@@ -221,9 +207,9 @@ curl -X POST http://127.0.0.1:8080/v1/pipelines/example-http-upload/upload \
|
||||
--data-binary @bundle.tar.gz
|
||||
```
|
||||
|
||||
Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md) for the copyable producer implementation guide.
|
||||
Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md).
|
||||
|
||||
The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`. Set `DISTRIBUTOR_EXAMPLE_UPLOAD_PIPELINE_ID` or pass a second argument to use another configured upload pipeline. It generates an idempotency key by default; set `DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY` when retrying the same producer run across separate process runs.
|
||||
The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`.
|
||||
|
||||
```sh
|
||||
go run ./examples/upload-client
|
||||
@@ -245,9 +231,9 @@ Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Co
|
||||
|
||||
Upload admission is bounded by `server.http.queue_size`. Publication concurrency is bounded by `server.http.max_concurrency`, and the coordinator does not run two uploads for the same pipeline at the same time.
|
||||
|
||||
`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`. If another request with the same key is still being staged before its manifest is known, the server returns a retryable `409 Conflict`. Idempotency records are memory-only and expire with completed upload status records.
|
||||
`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`.
|
||||
|
||||
The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size. The implementation also caps extracted file count.
|
||||
The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size.
|
||||
|
||||
The default bind address is private loopback. Put TLS, public routing, rate limiting, and external access policy in a reverse proxy or deployment layer.
|
||||
|
||||
@@ -267,7 +253,7 @@ S3 execution uses the AWS SDK for Go v2. See [S3-Compatible Storage Integration]
|
||||
|
||||
When explicit S3 credential variable names are configured, both must resolve to non-empty values through the process environment or `secrets.directory`. When omitted, the AWS SDK default credential chain is used as-is.
|
||||
|
||||
Normal single-owner replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Shared-root replacement deletes only current-owner omitted output objects and rewrites the shared state object. Merge publication retains omitted managed objects and may overwrite existing managed objects. Forced replacement deletes objects under the bounded destination bundle prefix. Distributor does not manage bucket versioning or delete markers.
|
||||
Normal managed replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Forced replacement deletes objects under the bounded destination bundle prefix and then writes schema version `4` catalog state. Distributor does not manage bucket versioning or delete markers.
|
||||
|
||||
## Secrets Operation
|
||||
|
||||
@@ -282,11 +268,10 @@ Use these recovery boundaries:
|
||||
- For source validation failures, regenerate the source bundle and manifest together.
|
||||
- For an empty or missing destination, rerun after fixing config or storage access.
|
||||
- For unmanaged destination content, move unrelated files aside or use a different destination path before publishing.
|
||||
- For shared-root ownership conflicts, change one owner so it writes a different destination path, or use a separate destination root.
|
||||
- For invalid or unsupported destination state, inspect `.distributor.json`; use `--force` only after dry-run review confirms bounded replacement is intended.
|
||||
- For missing managed output files recorded in state, run `reconcile-state --dry-run`, then apply `reconcile-state` if the missing files should no longer be considered managed.
|
||||
- For configured retention cleanup, run `prune --dry-run`, then apply `prune --apply` after reviewing the managed output list.
|
||||
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`. In merge mode, retained outputs may be intentional managed outputs from the prior state.
|
||||
- For state conflicts, verify the source, pipeline, destination, and existing `.distributor.json` before considering `--force`.
|
||||
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`.
|
||||
- For HTTP upload failures, inspect `/runs/<run-id>` while retained; after expiry or restart, rely on destination state and logs/output from the publishing run.
|
||||
|
||||
Do not edit `.distributor.json` during normal recovery. Treat it as the managed state record used for comparison and safe cleanup.
|
||||
|
||||
@@ -96,18 +96,15 @@ The source manifest should remain minimal. Routing, destination selection, publi
|
||||
|
||||
Each destination bundle path is managed by `.distributor.json`. This file is both the destination sentinel and the destination state record.
|
||||
|
||||
Publish execution writes single-owner or shared-root destination state according to destination `state.mode`. In shared-root state, one `.distributor.json` records multiple pipeline/destination owners and every managed output carries its owner identity.
|
||||
Publish execution writes catalog destination state. One `.distributor.json` records all managed outputs under the destination bundle path, and each output carries its owning pipeline id and destination id.
|
||||
|
||||
Single-owner state records:
|
||||
Catalog state records:
|
||||
|
||||
- `distributor` state schema version;
|
||||
- pipeline id;
|
||||
- destination id;
|
||||
- publication timestamp;
|
||||
- state creation and update timestamps;
|
||||
- single-owner state mode;
|
||||
- reconciliation mode;
|
||||
- the normalized source manifest used for publication;
|
||||
- catalog state mode;
|
||||
- owner identity for each managed output;
|
||||
- compact source identity for each managed output;
|
||||
- metadata for copied source outputs;
|
||||
- metadata for generated outputs, such as HTML files;
|
||||
- optional URL metadata for published outputs;
|
||||
@@ -117,40 +114,23 @@ A representative destination state file is:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 2,
|
||||
"schema_version": 4,
|
||||
"distributor_version": "0.1.0",
|
||||
"pipeline_id": "weather-daily",
|
||||
"destination_id": "static-html",
|
||||
"published_at": "2026-05-30T11:12:00Z",
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z",
|
||||
"state": {
|
||||
"mode": "single_owner"
|
||||
},
|
||||
"reconciliation": {
|
||||
"mode": "replace"
|
||||
},
|
||||
"source": {
|
||||
"manifest": {
|
||||
"schema_version": 1,
|
||||
"id": "weather.daily.brentwood.2026-05-30",
|
||||
"digest": "sha256:...",
|
||||
"created": "2026-05-30T11:10:00Z",
|
||||
"files": [
|
||||
{
|
||||
"path": "report.md",
|
||||
"sha256": "sha256:...",
|
||||
"size": 12345
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"primary_url": "https://reports.example.com/weather-daily/"
|
||||
"mode": "catalog"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"path": "index.html",
|
||||
"pipeline_id": "weather-daily",
|
||||
"destination_id": "static-html",
|
||||
"source": {
|
||||
"id": "weather.daily.brentwood.2026-05-30",
|
||||
"digest": "sha256:...",
|
||||
"created": "2026-05-30T11:10:00Z"
|
||||
},
|
||||
"kind": "generated",
|
||||
"source_path": "report.md",
|
||||
"transform": "markdown_to_html",
|
||||
@@ -167,13 +147,11 @@ A representative destination state file is:
|
||||
Destination comparison rules are based on `.distributor.json`:
|
||||
|
||||
- No `.distributor.json`: publish normally only if the destination bundle path is empty.
|
||||
- Existing state embeds the same normalized source manifest: skip as already published.
|
||||
- Existing state has the same source id and an older source `created`: replace, subject to destructive-operation safety rules.
|
||||
- Existing state has the same source id and a newer source `created`: skip because the destination is newer than the source.
|
||||
- Existing state has the same source id and same `created` but different digest: fail as a conflict.
|
||||
- Existing state has a different source id: fail as a conflict.
|
||||
|
||||
For older destination state, destination `reconciliation.mode` controls output updates. `replace` rewrites the managed output set to match the new plan. `merge` retains omitted managed outputs, overwrites only existing managed paths, and rejects planned paths that collide with unmanaged storage content.
|
||||
- Existing catalog state with additive workflow: write planned outputs and retain unrelated managed outputs.
|
||||
- Existing catalog state with replacement workflow: write planned outputs and remove omitted outputs for the current pipeline and destination owner.
|
||||
- Planned paths that collide with unmanaged storage content fail by default.
|
||||
- Invalid or unsupported destination state fails by default.
|
||||
- Explicit forced replacement may clear the bounded destination bundle path after dry-run review.
|
||||
|
||||
## Publication and Transform Policy
|
||||
|
||||
@@ -200,7 +178,7 @@ Application logic must interact with storage through internal backend interfaces
|
||||
|
||||
Adapters should be thin. Backend adapters should implement storage operations and translate backend-specific errors, but should not make bundle comparison, transform, routing, or replacement decisions.
|
||||
|
||||
Remote file-transfer support should prefer native protocol implementations over shelling out, unless a later design document records a reason to differ.
|
||||
Remote file copy support should prefer native protocol implementations over shelling out, unless a later design document records a reason to differ.
|
||||
|
||||
## Dependency Policy
|
||||
|
||||
@@ -221,7 +199,7 @@ Use this current layout unless the project has a documented reason to differ:
|
||||
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
|
||||
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
|
||||
- `internal/bundle`: storage-backed source bundle discovery and validation over the public manifest contract.
|
||||
- `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
|
||||
- `internal/state`: `.distributor.json` catalog parsing, validation, and output metadata.
|
||||
- `internal/link`: shared HTTP URL validation for configured and persisted link metadata.
|
||||
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
|
||||
- `internal/adapters/local`: local filesystem backend.
|
||||
@@ -229,7 +207,7 @@ Use this current layout unless the project has a documented reason to differ:
|
||||
- `internal/adapters/s3`: S3-compatible object storage backend.
|
||||
- `internal/transform`: transform interfaces, registry, planning, and shared transform models.
|
||||
- `internal/transform/markdown`: Markdown-to-HTML implementation.
|
||||
- `internal/publish`: destination planning, reconciliation, safety checks, and publish execution.
|
||||
- `internal/publish`: destination planning, catalog workflow safety checks, and publish execution.
|
||||
- `internal/notify`: notification interface and MVP no-op notifier.
|
||||
- `internal/logging`: logging setup and shared logging helpers.
|
||||
|
||||
@@ -264,7 +242,7 @@ Pipeline configuration should express:
|
||||
- per-destination transform policy;
|
||||
- per-destination public link policy;
|
||||
- validation behavior;
|
||||
- destination conflict/replacement behavior.
|
||||
- per-destination workflow and retention behavior.
|
||||
|
||||
## Modules and Registries
|
||||
|
||||
@@ -294,7 +272,7 @@ Errors should be actionable and preserve context. Wrap errors with operation, pi
|
||||
|
||||
Errors and logs must not expose secrets.
|
||||
|
||||
Use structured logging where practical. Logs should describe discovery, validation, planned actions, skipped transfers, conflicts, replacements, external calls, retries, and failure causes, but should not include large report contents by default.
|
||||
Use structured logging where practical. Logs should describe discovery, validation, planned actions, skipped copies, conflicts, replacements, external calls, retries, and failure causes, but should not include large report contents by default.
|
||||
|
||||
Skip and no-op decisions should be logged at an appropriate level so operators can distinguish successful publication from intentional no-op behavior.
|
||||
|
||||
|
||||
@@ -12,13 +12,13 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
|
||||
- `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
|
||||
- `internal/config`: YAML configuration structs, loading, defaults, and validation.
|
||||
- `internal/bundle`: storage-backed source bundle discovery and validation using the public manifest contract.
|
||||
- `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
|
||||
- `internal/state`: destination `.distributor.json` catalog parsing, validation, and output metadata.
|
||||
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
|
||||
- `internal/adapters/local`: local filesystem backend.
|
||||
- `internal/adapters/ssh`: SSH/SFTP backend.
|
||||
- `internal/adapters/s3`: S3-compatible object storage backend.
|
||||
- `internal/storage/fake`: in-memory backend for tests.
|
||||
- `internal/publish`: destination inspection, output planning, reconciliation, execution, managed cleanup, and explicit forced replacement.
|
||||
- `internal/publish`: destination inspection, output planning, catalog workflow planning, execution, managed cleanup, and explicit forced replacement.
|
||||
- `internal/transform`: transform interface and registry.
|
||||
- `internal/transform/markdown`: Markdown-to-HTML transform.
|
||||
- `internal/notify`: notification interface and current no-op notifier.
|
||||
|
||||
@@ -43,6 +43,7 @@ Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- public HTTP API reference: `docs/api.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
@@ -122,6 +123,22 @@ Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Public HTTP API service
|
||||
|
||||
Required:
|
||||
- `docs/api.md`
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/consumers/`, for task-oriented client integration guides
|
||||
- `docs/integrations/`, for upstream/downstream service contracts
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Project with public packages or consumer APIs
|
||||
|
||||
Required:
|
||||
@@ -173,6 +190,32 @@ It should include:
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
|
||||
### docs/api.md
|
||||
|
||||
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||
|
||||
Required for projects whose primary public interface is HTTP.
|
||||
|
||||
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||
|
||||
It should include:
|
||||
|
||||
1. base URL conventions;
|
||||
2. authentication and authorization behavior, if implemented;
|
||||
3. response envelope;
|
||||
4. supported media types and content negotiation behavior;
|
||||
5. shared query parameters;
|
||||
6. endpoint reference grouped by route family;
|
||||
7. request parameters and validation rules;
|
||||
8. response fields, units, nullability, and optionality;
|
||||
9. error response shape and status codes;
|
||||
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||
11. compact request and response examples.
|
||||
|
||||
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||
|
||||
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
@@ -264,6 +307,8 @@ Required for projects with public packages, SDKs, client APIs, plugin APIs, or o
|
||||
|
||||
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||
|
||||
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||
|
||||
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||
|
||||
1. intended consumer audience and use cases;
|
||||
@@ -330,6 +375,8 @@ Required for projects that depend on external CLIs, APIs, services, protocols, o
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||
|
||||
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
## Examples Directory
|
||||
@@ -385,9 +432,10 @@ Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Consumer-facing docs explain public APIs without duplicating integration contracts.
|
||||
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
|
||||
@@ -93,10 +93,10 @@ internal docs.
|
||||
- Current upload status, queue, and idempotency state are memory-only.
|
||||
- Producers submit complete tar or gzip-compressed tar source bundles today.
|
||||
- Producers do not choose destination ids, destination paths, transforms, links,
|
||||
publish policy, transfer policy, storage backends, reconciliation mode, state
|
||||
mode, or retention policy through upload requests.
|
||||
publish policy, destination workflow, storage backends, or retention policy
|
||||
through upload requests.
|
||||
- Source manifests remain free of routing, destination, transform, credential,
|
||||
reconciliation, state, and retention data.
|
||||
workflow, state, and retention data.
|
||||
- Public access policy, TLS termination, and rate limiting belong outside
|
||||
`distributor` unless a future implementation changes that boundary.
|
||||
- `distributor` is not a broad storage synchronization tool unless a future
|
||||
|
||||
@@ -24,7 +24,9 @@ Reference: [Configuration](config.md#config-file-loading).
|
||||
|
||||
Symptom: `parse config ... field not found`.
|
||||
|
||||
Likely cause: the YAML contains a key that is not part of the implemented config schema.
|
||||
Likely cause: the YAML contains a key that is not part of the implemented
|
||||
config schema. Pre-workflow destination policy keys for state mode, conflict
|
||||
handling, ownership adoption, or per-comparison copy decisions are rejected.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -32,9 +34,9 @@ Diagnostic:
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: remove or rename unsupported fields using the canonical config reference.
|
||||
Safe fix: remove unsupported fields using the canonical config reference. Destination behavior is configured with `workflow`, `publish`, `transform`, `path_mapping`, `links`, and `retention`.
|
||||
|
||||
Reference: [Configuration](config.md).
|
||||
Reference: [Configuration](config.md#destination-fields).
|
||||
|
||||
## Backend Name Or Placement Is Invalid
|
||||
|
||||
@@ -71,9 +73,9 @@ Reference: [CLI](cli.md#validate).
|
||||
|
||||
## Reconcile-State Selector Is Missing Or Wrong
|
||||
|
||||
Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
|
||||
Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "<id>" not found`, or `destination <id> not found`.
|
||||
|
||||
Likely cause: the command did not identify one configured destination root, or the selected root contains state for a different single-owner pipeline/destination.
|
||||
Likely cause: the command did not identify one configured destination root.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -83,15 +85,15 @@ rg -n 'pipelines:|destinations:|id:' <config-path>
|
||||
cat <destination-path>/.distributor.json
|
||||
```
|
||||
|
||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. For unrelated single-owner state, use the correct config selector or a separate destination root; `reconcile-state` will not take over mismatched state.
|
||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file.
|
||||
|
||||
Reference: [CLI](cli.md#reconcile-state).
|
||||
|
||||
## Prune Selector Or Mode Is Missing Or Wrong
|
||||
|
||||
Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
|
||||
Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "<id>" not found`, or `destination <id> not found`.
|
||||
|
||||
Likely cause: the command did not identify one configured destination root, did not choose exactly one execution mode, or the selected root contains state for a different single-owner pipeline/destination.
|
||||
Likely cause: the command did not identify one configured destination root or did not choose exactly one execution mode.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -101,7 +103,7 @@ rg -n 'retention:|prune:|pipelines:|destinations:|id:' <config-path>
|
||||
cat <destination-path>/.distributor.json
|
||||
```
|
||||
|
||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both. For unrelated single-owner state, use the correct config selector or a separate destination root; `prune` will not take over mismatched state.
|
||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both.
|
||||
|
||||
Reference: [CLI](cli.md#prune).
|
||||
|
||||
@@ -187,9 +189,9 @@ Reference: [Operations](operations.md#cleanup-and-recovery).
|
||||
|
||||
## Destination Has Unmanaged Content
|
||||
|
||||
Symptom: `destination has content but no distributor state` or a plan reason containing `fail_unmanaged`.
|
||||
Symptom: `destination has content but no distributor state`, `destination output path ... exists but is not managed by catalog state`, or a plan reason containing `fail_unmanaged`.
|
||||
|
||||
Likely cause: the destination bundle path contains files but no valid `.distributor.json`, so `distributor` will not claim it by default.
|
||||
Likely cause: the destination bundle path contains files but no valid `.distributor.json`, or a planned output path collides with storage content that valid catalog state does not record. `distributor` will not claim unmanaged content by default.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -197,26 +199,43 @@ Diagnostic:
|
||||
find <destination-path> -maxdepth 2 -print
|
||||
```
|
||||
|
||||
Safe fix: choose an empty destination path, move unrelated files aside, or preview `run --dry-run --force` only after confirming the reported bundle path is safe to replace.
|
||||
Safe fix: choose an empty destination path, move unrelated files aside, or preview `run --dry-run --force` only after confirming the reported destination bundle path is safe to replace.
|
||||
|
||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||
|
||||
## Destination State Conflicts With Source
|
||||
## Destination State Is Invalid Or Unsupported
|
||||
|
||||
Symptom: `fail_conflict`, `destination source id differs`, `same id and created time but different digest`, `pipeline id ... does not match`, or `destination id ... does not match`.
|
||||
Symptom: `fail_conflict`, `parse distributor state`, `state schema_version must be 4`, or `unsupported future destination state`.
|
||||
|
||||
Likely cause: `.distributor.json` belongs to a different pipeline, destination, source id, or same-created source with different content.
|
||||
Likely cause: `.distributor.json` is invalid JSON, has invalid catalog fields, or uses an unsupported future schema.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
cat <destination-path>/.distributor.json
|
||||
go run ./cmd/distributor inspect <source-root>
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run --format json
|
||||
```
|
||||
|
||||
Safe fix: verify the source and destination are intended to match. Use a separate destination path for unrelated content. To replace the existing state, configure `transfer.on_conflict: replace`, preview with `--dry-run --force`, then publish with `--force`.
|
||||
Safe fix: restore a valid catalog state file from backup, choose a different destination path, or use `--force` only after `run --dry-run --force` reports the intended bounded `force_replace`.
|
||||
|
||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||
|
||||
## Destination Uses Superseded Legacy State
|
||||
|
||||
Symptom: dry-run reports a normal catalog action against an older `.distributor.json`, or `reconcile-state` / `prune` reports that the destination state schema is superseded.
|
||||
|
||||
Likely cause: the destination contains a state file written by an older implementation. Publish planning can replace it with catalog state on successful `run`, but maintenance commands only operate on current catalog state.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
cat <destination-path>/.distributor.json
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: preview the publish plan, then run publication if the destination path is correct. The successful run writes schema version `4` catalog state.
|
||||
|
||||
Reference: [Destination State Contract](integrations/destination-state.md).
|
||||
|
||||
## Destination State References Missing Managed Outputs
|
||||
|
||||
@@ -242,7 +261,7 @@ Reference: [Operations](operations.md#destination-state-repair).
|
||||
|
||||
Symptom: `prune --dry-run` reports `planned=0` or JSON `planned_outputs: []`.
|
||||
|
||||
Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected shared-root owner has no eligible outputs.
|
||||
Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected pipeline/destination owner has no eligible outputs.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -260,27 +279,11 @@ Safe fix: verify the selected destination's retention policy and the managed out
|
||||
|
||||
Reference: [Operations](operations.md#managed-output-pruning).
|
||||
|
||||
## Destination Is Newer Than Source
|
||||
|
||||
Symptom: `skip_destination_newer` or `destination is newer and replacement requires --force`.
|
||||
|
||||
Likely cause: the destination state records a source manifest with a later `created` timestamp than the current source.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run --format json
|
||||
```
|
||||
|
||||
Safe fix: keep the default skip behavior unless replacement is intentional. To replace newer state, configure `transfer.on_destination_newer: replace`, preview with `--dry-run --force`, then publish with `--force`.
|
||||
|
||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||
|
||||
## Forced Replacement Appears In A Plan
|
||||
|
||||
Symptom: dry-run output includes `force_replace`.
|
||||
|
||||
Likely cause: the run used `--force`, and planning selected a supported destructive replacement.
|
||||
Likely cause: the run used `--force`, and catalog planning selected a supported destructive replacement for unmanaged destination content, a planned unmanaged path collision, invalid destination state, or unsupported future destination state.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -288,15 +291,15 @@ Diagnostic:
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run --force
|
||||
```
|
||||
|
||||
Safe fix: inspect the pipeline id, destination id, backend, and bundle path. Proceed only if deleting everything inside that destination bundle path is intended.
|
||||
Safe fix: inspect the pipeline id, destination id, backend, and bundle path. Confirm the `destination_path` in JSON output, or the fixed-path `target=.` text output, before applying. Proceed only if deleting everything inside that destination bundle path is intended; `force_replace` then writes planned outputs and schema version `4` catalog state.
|
||||
|
||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||
|
||||
## Output Path Collision
|
||||
|
||||
Symptom: `destination output path collision` or `merge output path ... exists but is not managed by destination state`.
|
||||
Symptom: `destination output path collision` or `destination output path ... exists but is not managed by catalog state`.
|
||||
|
||||
Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`. For merge reconciliation, it can also mean a planned output path already exists in storage but is not recorded in `.distributor.json` as managed.
|
||||
Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`, or a planned output path already exists in storage but is not catalog-managed.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -304,44 +307,10 @@ Diagnostic:
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For merge reconciliation, move unmanaged content aside, choose another destination path, or use replacement/forced replacement only when deleting the destination bundle path is intended.
|
||||
Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For unmanaged storage collisions, move the unmanaged file aside, choose another destination path, or use forced replacement only when deleting the destination bundle path is intended.
|
||||
|
||||
Reference: [Configuration](config.md#publish-and-transform-policy).
|
||||
|
||||
## Shared-Root Ownership Conflict
|
||||
|
||||
Symptom: `fail_conflict` with a reason like `destination output path ... is owned by <pipeline>/<destination>`.
|
||||
|
||||
Likely cause: a `state.mode: shared_root` destination planned an output path already recorded in `.distributor.json` for another pipeline/destination owner.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
cat <destination-path>/.distributor.json
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: change one owner so it writes a different output path, use a separate destination root, or intentionally replace the whole destination root only after previewing with `--dry-run --force`.
|
||||
|
||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
||||
|
||||
## Shared-Root Unmanaged Path Collision
|
||||
|
||||
Symptom: `fail_unmanaged` with a reason like `destination output path ... exists but is not managed by destination state`.
|
||||
|
||||
Likely cause: a `state.mode: shared_root` destination planned a new output path that already exists in storage but is not recorded as managed in `.distributor.json`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
find <destination-path> -maxdepth 2 -print
|
||||
cat <destination-path>/.distributor.json
|
||||
```
|
||||
|
||||
Safe fix: move the unmanaged file aside, change the planned output path, or use forced replacement only when deleting the configured destination root is intended.
|
||||
|
||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||
|
||||
## Run Failed After Writing Some Files
|
||||
|
||||
Symptom: a destination write failed and the command exited non-zero after partial work.
|
||||
@@ -354,9 +323,9 @@ Diagnostic:
|
||||
find <destination-path> -maxdepth 2 -print
|
||||
```
|
||||
|
||||
Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. In merge mode, previously managed retained or overwritten outputs may remain intentionally. Rerun `--dry-run` before publishing again.
|
||||
Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. Rerun `--dry-run` before publishing again.
|
||||
|
||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
||||
Reference: [Operations](operations.md#catalog-publish-behavior).
|
||||
|
||||
## JSON Run Result Has `ok: false`
|
||||
|
||||
@@ -378,240 +347,14 @@ Reference: [CLI](cli.md#output-and-exit-behavior).
|
||||
|
||||
Symptom: `load secrets directory ... no such file or directory`, `permission denied`, or `secret filename ... is invalid`.
|
||||
|
||||
Likely cause: `secrets.directory` points to a missing or unreadable directory, or it contains a regular file whose name is not a valid credential variable name.
|
||||
Likely cause: `secrets.directory` points to a missing or unreadable directory, or contains a filename that cannot be used as a credential variable name.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ls -ld <secrets-directory>
|
||||
find <secrets-directory> -maxdepth 1 -type f -printf '%f\n'
|
||||
ls -la <secrets-directory>
|
||||
```
|
||||
|
||||
Safe fix: mount or create the directory, adjust permissions for the service user, or rename/remove invalid secret files. Secret filenames must match `[A-Za-z_][A-Za-z0-9_]*`.
|
||||
Safe fix: create the directory, fix permissions, or rename secret files to valid environment-variable-style names.
|
||||
|
||||
Reference: [Configuration](config.md#secrets).
|
||||
|
||||
## Credential Variable Is Missing Or Empty
|
||||
|
||||
Symptom: `credential environment variable ... is not set`, `credential environment variable ... is empty`, or S3 authentication errors such as `AccessDenied`, `InvalidAccessKeyId`, or `SignatureDoesNotMatch`.
|
||||
|
||||
Likely cause: configured S3 credential variable names are not available through the process environment or `secrets.directory`, are empty, or do not authorize the requested bucket/prefix.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
env | cut -d= -f1 | rg '^(<access-key-variable>|<secret-key-variable>)$'
|
||||
ls -l <secrets-directory>
|
||||
```
|
||||
|
||||
Safe fix: provide both configured S3 credential values, correct IAM/service permissions, or omit explicit credential fields to use the AWS SDK default credential chain.
|
||||
|
||||
Reference: [Configuration](config.md#s3-compatible-backend).
|
||||
|
||||
## Secret File Is Ignored In Favor Of Environment
|
||||
|
||||
Symptom: `secret ... ignored because the real environment already has that variable`.
|
||||
|
||||
Likely cause: the same credential name exists in the process environment and `secrets.directory` with different values.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
env | cut -d= -f1 | rg '^<variable-name>$'
|
||||
ls -l <secrets-directory>/<variable-name>
|
||||
```
|
||||
|
||||
Safe fix: remove one source of the credential or intentionally keep the process environment value. `distributor` does not print either value.
|
||||
|
||||
Reference: [Operations](operations.md#secrets-operation).
|
||||
|
||||
## SSH Auth Is Not Configured
|
||||
|
||||
Symptom: `no SSH auth methods configured`.
|
||||
|
||||
Likely cause: no SSH agent is available and `ssh_key_file` is missing or unreadable.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
test -n "$SSH_AUTH_SOCK" && ssh-add -l
|
||||
ls -l <ssh-key-file>
|
||||
```
|
||||
|
||||
Safe fix: start an SSH agent with an appropriate key loaded, or configure a readable private key with `ssh_key_file`.
|
||||
|
||||
Reference: [Configuration](config.md#sshsftp-backend).
|
||||
|
||||
## SSH Host Key Fails
|
||||
|
||||
Symptom: `host key ... is unknown`, `known_hosts is required`, or `host key ... has changed`.
|
||||
|
||||
Likely cause: strict host key checking has no trusted key, `accept-new` cannot persist a new key, or the remote host key differs from the stored key.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ls -l <known-hosts-path>
|
||||
ssh-keygen -F <host> -f <known-hosts-path>
|
||||
```
|
||||
|
||||
Safe fix: pre-populate `known_hosts` for `strict`, configure a writable `known_hosts` path for `accept-new`, or verify the server identity before updating a changed key. Do not disable host key checking to bypass an unexpected changed key.
|
||||
|
||||
Reference: [Operations](operations.md#sshsftp).
|
||||
|
||||
## S3 Prefix Is Invalid
|
||||
|
||||
Symptom: `prefix must be a clean relative slash-separated path`.
|
||||
|
||||
Likely cause: the S3 prefix contains traversal, dot segments, empty segments, or backslashes after leading and trailing slashes are trimmed.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: use a clean relative prefix such as `reports/archive`, or omit `prefix`.
|
||||
|
||||
Reference: [Configuration](config.md#s3-compatible-backend).
|
||||
|
||||
## S3 Location Or Connectivity Fails
|
||||
|
||||
Symptom: `NoSuchBucket`, `InvalidBucketName`, `not_found`, endpoint connection failures, or TLS/network errors.
|
||||
|
||||
Likely cause: endpoint, bucket, prefix, region, path-style mode, network routing, or credentials are wrong for the service.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
curl -I <endpoint>
|
||||
```
|
||||
|
||||
Safe fix: verify `endpoint`, `bucket`, `region`, `prefix`, and `force_path_style`. For S3-compatible services, keep `force_path_style: true` unless the service requires virtual-host addressing. Distributor does not provide an insecure TLS bypass setting.
|
||||
|
||||
Reference: [Operations](operations.md#s3-compatible-storage).
|
||||
|
||||
## HTTP Server Cannot Bind
|
||||
|
||||
Symptom: `bind HTTP server ... address already in use`.
|
||||
|
||||
Likely cause: another process is listening on `server.http.bind`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
ss -ltnp | rg '<port>'
|
||||
```
|
||||
|
||||
Safe fix: stop the conflicting process or configure a different bind address.
|
||||
|
||||
Reference: [Configuration](config.md#serverhttp).
|
||||
|
||||
## HTTP Upload Token Is Missing Or Duplicated
|
||||
|
||||
Symptom: `upload token environment variable ... is not set`, `... is empty`, or `upload token environment variables ... resolve to the same value`.
|
||||
|
||||
Likely cause: a top-level upload token record references a missing or empty `token_env`, or two token records resolve to the same bearer token.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
rg -n 'token_env:' <config-path>
|
||||
env | cut -d= -f1 | rg '^<token-variable>$'
|
||||
ls -l <secrets-directory>/<token-variable>
|
||||
```
|
||||
|
||||
Safe fix: provide one distinct non-empty token value per upload token record through the process environment or `secrets.directory`. Do not put literal tokens in YAML.
|
||||
|
||||
Reference: [Configuration](config.md#upload_tokens).
|
||||
|
||||
## Upload Request Is Unauthorized
|
||||
|
||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `401`.
|
||||
|
||||
Likely cause: the request lacks `Authorization: Bearer <token>`, has an empty token, or uses a token that does not match any configured upload token record.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
curl -i -X POST http://127.0.0.1:8080/v1/pipelines/<pipeline-id>/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/x-tar" \
|
||||
--data-binary @bundle.tar
|
||||
```
|
||||
|
||||
Safe fix: use the token value resolved by the configured `token_env`. Do not include token values in logs or tickets.
|
||||
|
||||
Reference: [Operations](operations.md#http-upload-operation).
|
||||
|
||||
## Upload Request Is Forbidden
|
||||
|
||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `403`.
|
||||
|
||||
Likely cause: the bearer token is valid, but its configured `allow_pipelines` list does not include the requested upload pipeline.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
rg -n 'upload_tokens:|allow_pipelines:|id:' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: request the intended pipeline id, or update the token allowlist to include the configured `http_upload` pipeline that this producer may submit to.
|
||||
|
||||
Reference: [Configuration](config.md#upload_tokens).
|
||||
|
||||
## Upload Request Is Rejected Before A Run ID
|
||||
|
||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `400`, `413`, `415`, or `503`.
|
||||
|
||||
Likely cause: the request path has an invalid pipeline id, included a `pipeline` or `pipeline_id` query, archive content is malformed, the body exceeds size limits, content type is unsupported, or the in-memory upload queue is full.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
tar -tf bundle.tar
|
||||
tar -tzf bundle.tar.gz
|
||||
rg -n 'max_upload_size|queue_size|max_concurrency' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: send one valid tar or tar.gz source bundle archive to `/v1/pipelines/<pipeline-id>/upload` with `Content-Type: application/x-tar`, `application/gzip`, or `application/x-gzip`; remove pipeline query parameters; reduce archive size or raise the configured limit; retry after queue pressure drops.
|
||||
|
||||
Reference: [Operations](operations.md#http-upload-operation).
|
||||
|
||||
## Upload Idempotency Conflict
|
||||
|
||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `409`.
|
||||
|
||||
Likely cause: the request reused an `Idempotency-Key` for the same token id and pipeline id with a different source manifest, or another request with the same key is still being staged before its manifest is known.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
curl -i -X POST http://127.0.0.1:8080/v1/pipelines/<pipeline-id>/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/gzip" \
|
||||
-H "Idempotency-Key: <key>" \
|
||||
--data-binary @bundle.tar.gz
|
||||
```
|
||||
|
||||
Safe fix: if the response includes `"retryable":true`, retry the same upload later with the same key. Otherwise, inspect the producer operation and use the same key only for the same source bundle.
|
||||
|
||||
Reference: [HTTP Upload API Contract](integrations/http-upload.md).
|
||||
|
||||
## Upload Status Is Missing
|
||||
|
||||
Symptom: `GET /runs/<run_id>` returns `404`.
|
||||
|
||||
Likely cause: the run id is wrong, the process restarted, or the retained status record expired after `server.http.retention`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
curl -i http://127.0.0.1:8080/runs/<run-id>
|
||||
rg -n 'retention:' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: use the exact `run_id` returned by upload admission. Increase retention if operators need a longer status window.
|
||||
|
||||
Reference: [Operations](operations.md#http-upload-operation).
|
||||
|
||||
@@ -1,27 +1,25 @@
|
||||
pipelines:
|
||||
- id: example-shared-root-source
|
||||
- id: example-additive-source
|
||||
source:
|
||||
backend: local
|
||||
path: examples/source-bundle
|
||||
destinations:
|
||||
- id: shared-local-root
|
||||
- id: catalog-source
|
||||
backend: local
|
||||
path: workspace/published/shared-root
|
||||
state:
|
||||
mode: shared_root
|
||||
path: workspace/published/additive-workflow
|
||||
workflow: additive
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
- id: example-shared-root-html
|
||||
- id: example-additive-html
|
||||
source:
|
||||
backend: local
|
||||
path: examples/source-bundle
|
||||
destinations:
|
||||
- id: shared-local-root
|
||||
- id: catalog-html
|
||||
backend: local
|
||||
path: workspace/published/shared-root
|
||||
state:
|
||||
mode: shared_root
|
||||
path: workspace/published/additive-workflow
|
||||
workflow: additive
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
@@ -7,6 +7,7 @@ pipelines:
|
||||
- id: local-source-archive
|
||||
backend: local
|
||||
path: workspace/published/archive-and-latest/archive
|
||||
workflow: additive
|
||||
path_mapping:
|
||||
mode: preserve_relative
|
||||
publish:
|
||||
@@ -15,6 +16,7 @@ pipelines:
|
||||
- id: local-html-latest
|
||||
backend: local
|
||||
path: workspace/published/archive-and-latest/latest
|
||||
workflow: replacement
|
||||
path_mapping:
|
||||
mode: fixed
|
||||
links:
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
pipelines:
|
||||
- id: example-merge-reconciliation
|
||||
- id: example-replacement-workflow
|
||||
source:
|
||||
backend: local
|
||||
path: examples/source-bundle
|
||||
destinations:
|
||||
- id: local-merge-html
|
||||
- id: local-latest-html
|
||||
backend: local
|
||||
path: workspace/published/merge-reconciliation
|
||||
path: workspace/published/replacement-workflow
|
||||
workflow: replacement
|
||||
path_mapping:
|
||||
mode: fixed
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
reconciliation:
|
||||
mode: merge
|
||||
mode: index
|
||||
@@ -170,7 +170,7 @@ func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader
|
||||
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", written, opts.Size))
|
||||
}
|
||||
if opts.PreferAtomic {
|
||||
if err := b.client.Rename(writePath, nativePath); err != nil {
|
||||
if err := renamePromotedFile(b.client, writePath, nativePath, opts.Overwrite); err != nil {
|
||||
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
|
||||
}
|
||||
cleanup = false
|
||||
@@ -178,6 +178,27 @@ func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader
|
||||
return b.Stat(ctx, logicalPath)
|
||||
}
|
||||
|
||||
type sftpRenamer interface {
|
||||
PosixRename(oldname, newname string) error
|
||||
Rename(oldname, newname string) error
|
||||
Remove(path string) error
|
||||
}
|
||||
|
||||
func renamePromotedFile(client sftpRenamer, oldname, newname string, overwrite bool) error {
|
||||
if !overwrite {
|
||||
return client.Rename(oldname, newname)
|
||||
}
|
||||
if err := client.PosixRename(oldname, newname); err == nil {
|
||||
return nil
|
||||
} else if !isReplaceRenameFallbackError(err) {
|
||||
return err
|
||||
}
|
||||
if err := client.Remove(newname); err != nil && !isNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return client.Rename(oldname, newname)
|
||||
}
|
||||
|
||||
func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return storage.Entry{}, err
|
||||
@@ -464,6 +485,14 @@ func isNotExist(err error) bool {
|
||||
return errors.Is(err, fs.ErrNotExist) || errors.Is(err, os.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile)
|
||||
}
|
||||
|
||||
func isReplaceRenameFallbackError(err error) bool {
|
||||
if errors.Is(err, sftp.ErrSSHFxFailure) || errors.Is(err, sftp.ErrSSHFxOpUnsupported) {
|
||||
return true
|
||||
}
|
||||
var statusErr *sftp.StatusError
|
||||
return errors.As(err, &statusErr) && (statusErr.FxCode() == sftp.ErrSSHFxFailure || statusErr.FxCode() == sftp.ErrSSHFxOpUnsupported)
|
||||
}
|
||||
|
||||
func (b *Backend) translateError(op, logicalPath string, err error) error {
|
||||
kind := storage.ErrUnknown
|
||||
switch {
|
||||
|
||||
125
internal/adapters/ssh/backend_test.go
Normal file
125
internal/adapters/ssh/backend_test.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
)
|
||||
|
||||
func TestRenamePromotedFileUsesPlainRenameWithoutOverwrite(t *testing.T) {
|
||||
client := &recordingRenamer{}
|
||||
if err := renamePromotedFile(client, "temp", "index.html", false); err != nil {
|
||||
t.Fatalf("renamePromotedFile() error = %v", err)
|
||||
}
|
||||
if got, want := client.calls, []string{"rename temp index.html"}; !equalStrings(got, want) {
|
||||
t.Fatalf("calls = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePromotedFileUsesPosixRenameForOverwrite(t *testing.T) {
|
||||
client := &recordingRenamer{}
|
||||
if err := renamePromotedFile(client, "temp", "index.html", true); err != nil {
|
||||
t.Fatalf("renamePromotedFile() error = %v", err)
|
||||
}
|
||||
if got, want := client.calls, []string{"posix temp index.html"}; !equalStrings(got, want) {
|
||||
t.Fatalf("calls = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePromotedFileFallsBackWhenReplaceRenameUnsupported(t *testing.T) {
|
||||
for _, err := range []error{
|
||||
sftp.ErrSSHFxOpUnsupported,
|
||||
sftp.ErrSSHFxFailure,
|
||||
&sftp.StatusError{Code: uint32(sftp.ErrSSHFxOpUnsupported)},
|
||||
&sftp.StatusError{Code: uint32(sftp.ErrSSHFxFailure)},
|
||||
} {
|
||||
t.Run(err.Error(), func(t *testing.T) {
|
||||
client := &recordingRenamer{posixErr: err}
|
||||
if err := renamePromotedFile(client, "temp", "index.html", true); err != nil {
|
||||
t.Fatalf("renamePromotedFile() error = %v", err)
|
||||
}
|
||||
want := []string{"posix temp index.html", "remove index.html", "rename temp index.html"}
|
||||
if got := client.calls; !equalStrings(got, want) {
|
||||
t.Fatalf("calls = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePromotedFileIgnoresMissingTargetDuringFallback(t *testing.T) {
|
||||
client := &recordingRenamer{
|
||||
posixErr: sftp.ErrSSHFxOpUnsupported,
|
||||
removeErr: &os.PathError{
|
||||
Op: "remove",
|
||||
Path: "index.html",
|
||||
Err: os.ErrNotExist,
|
||||
},
|
||||
}
|
||||
if err := renamePromotedFile(client, "temp", "index.html", true); err != nil {
|
||||
t.Fatalf("renamePromotedFile() error = %v", err)
|
||||
}
|
||||
want := []string{"posix temp index.html", "remove index.html", "rename temp index.html"}
|
||||
if got := client.calls; !equalStrings(got, want) {
|
||||
t.Fatalf("calls = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePromotedFileDoesNotFallbackForPermissionError(t *testing.T) {
|
||||
client := &recordingRenamer{posixErr: sftp.ErrSSHFxPermissionDenied}
|
||||
if err := renamePromotedFile(client, "temp", "index.html", true); !errors.Is(err, sftp.ErrSSHFxPermissionDenied) {
|
||||
t.Fatalf("renamePromotedFile() error = %v, want permission denied", err)
|
||||
}
|
||||
if got, want := client.calls, []string{"posix temp index.html"}; !equalStrings(got, want) {
|
||||
t.Fatalf("calls = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenamePromotedFileReturnsRemoveFallbackError(t *testing.T) {
|
||||
client := &recordingRenamer{
|
||||
posixErr: sftp.ErrSSHFxOpUnsupported,
|
||||
removeErr: sftp.ErrSSHFxPermissionDenied,
|
||||
}
|
||||
if err := renamePromotedFile(client, "temp", "index.html", true); !errors.Is(err, sftp.ErrSSHFxPermissionDenied) {
|
||||
t.Fatalf("renamePromotedFile() error = %v, want permission denied", err)
|
||||
}
|
||||
want := []string{"posix temp index.html", "remove index.html"}
|
||||
if got := client.calls; !equalStrings(got, want) {
|
||||
t.Fatalf("calls = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingRenamer struct {
|
||||
calls []string
|
||||
posixErr error
|
||||
renameErr error
|
||||
removeErr error
|
||||
}
|
||||
|
||||
func (r *recordingRenamer) PosixRename(oldname, newname string) error {
|
||||
r.calls = append(r.calls, "posix "+oldname+" "+newname)
|
||||
return r.posixErr
|
||||
}
|
||||
|
||||
func (r *recordingRenamer) Rename(oldname, newname string) error {
|
||||
r.calls = append(r.calls, "rename "+oldname+" "+newname)
|
||||
return r.renameErr
|
||||
}
|
||||
|
||||
func (r *recordingRenamer) Remove(path string) error {
|
||||
r.calls = append(r.calls, "remove "+path)
|
||||
return r.removeErr
|
||||
}
|
||||
|
||||
func equalStrings(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for index := range a {
|
||||
if a[index] != b[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -211,29 +211,18 @@ func removePrunedStateRecords(ctx context.Context, backend storage.Backend, stat
|
||||
if len(paths) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
if document.SingleOwner != nil {
|
||||
next, changed := state.RemoveMissingOutputs(*document.SingleOwner, paths)
|
||||
if document.Catalog != nil {
|
||||
next, changed := state.RemoveMissingCatalogOwnerOutputs(*document.Catalog, scope, paths)
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
next.UpdatedAt = now
|
||||
if err := state.Validate(next); err != nil {
|
||||
if err := state.ValidateCatalog(next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, writeRepairedState(ctx, backend, statePath, next)
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
next, changed := state.RemoveMissingSharedRootOwnerOutputs(*document.SharedRoot, scope, paths)
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
next.UpdatedAt = now
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, writeRepairedState(ctx, backend, statePath, next)
|
||||
}
|
||||
return false, fmt.Errorf("destination state document is empty")
|
||||
return false, unsupportedStateDocumentError(document)
|
||||
}
|
||||
|
||||
func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options PrunePlanOptions) (PrunePlanReport, error) {
|
||||
@@ -266,17 +255,17 @@ func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options
|
||||
}
|
||||
|
||||
func pruneCandidatesForDocument(document state.StateDocument, scope state.OwnerScope) ([]state.PruneCandidate, error) {
|
||||
if document.SingleOwner != nil {
|
||||
singleOwner := *document.SingleOwner
|
||||
if singleOwner.PipelineID != scope.PipelineID || singleOwner.DestinationID != scope.DestinationID {
|
||||
return nil, fmt.Errorf("state owner is %s/%s, not %s/%s", singleOwner.PipelineID, singleOwner.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
if document.Catalog != nil {
|
||||
return state.CatalogPruneCandidates(*document.Catalog, scope), nil
|
||||
}
|
||||
return state.SingleOwnerPruneCandidates(singleOwner), nil
|
||||
return nil, unsupportedStateDocumentError(document)
|
||||
}
|
||||
|
||||
func unsupportedStateDocumentError(document state.StateDocument) error {
|
||||
if document.SupersededLegacy != nil {
|
||||
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
return state.SharedRootPruneCandidates(*document.SharedRoot, scope), nil
|
||||
}
|
||||
return nil, fmt.Errorf("destination state document is empty")
|
||||
return fmt.Errorf("destination state document is empty")
|
||||
}
|
||||
|
||||
func pruneOlderThan(policy config.PrunePolicy) *time.Duration {
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
)
|
||||
|
||||
func TestPlanPruneDisabledPolicy(t *testing.T) {
|
||||
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
|
||||
document := state.StateDocument{Catalog: &state.CatalogState{}}
|
||||
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
@@ -28,12 +28,12 @@ func TestPlanPruneDisabledPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
|
||||
func TestPlanPruneCatalogOutputs(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
olderThan := config.Duration(48 * time.Hour)
|
||||
destinationState := pruneSingleOwnerState(now)
|
||||
catalog := pruneCatalogState(now)
|
||||
|
||||
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
|
||||
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
|
||||
Enabled: true,
|
||||
OlderThan: &olderThan,
|
||||
}, PrunePlanOptions{
|
||||
@@ -52,12 +52,12 @@ func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
|
||||
func TestPlanPruneCatalogCurrentOwnerOnly(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
keepLatest := 0
|
||||
sharedRoot := pruneSharedRootState(now)
|
||||
catalog := pruneCatalogState(now)
|
||||
|
||||
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
|
||||
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
|
||||
Enabled: true,
|
||||
KeepLatest: &keepLatest,
|
||||
}, PrunePlanOptions{
|
||||
@@ -68,10 +68,10 @@ func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("PlanPrune() error = %v", err)
|
||||
}
|
||||
if got, want := report.CheckedCount, 1; got != want {
|
||||
if got, want := report.CheckedCount, 2; got != want {
|
||||
t.Fatalf("checked count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
|
||||
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt,fresh.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -80,8 +80,8 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
original := pruneSingleOwnerState(now)
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, original)
|
||||
original := pruneCatalogState(now)
|
||||
writeFakeCatalogState(t, backend, original)
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -102,12 +102,12 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
|
||||
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "old.txt,fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
|
||||
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
|
||||
if !catalog.UpdatedAt.Equal(original.UpdatedAt) {
|
||||
t.Fatalf("state updated_at = %s, want original %s", catalog.UpdatedAt, original.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -134,14 +134,18 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
assertFakeStateExists(t, backend)
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
if !destinationState.UpdatedAt.Equal(now) {
|
||||
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion {
|
||||
t.Fatalf("state schema_version = %d, want %d", catalog.SchemaVersion, state.CatalogSchemaVersion)
|
||||
}
|
||||
if !catalog.UpdatedAt.Equal(now) {
|
||||
t.Fatalf("state updated_at = %s, want %s", catalog.UpdatedAt, now)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,7 +154,7 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
||||
backend := fake.New()
|
||||
keepLatest := 0
|
||||
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -170,18 +174,17 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
assertFakeStateExists(t, backend)
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want only failed output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
||||
func TestPrunePreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
keepLatest := 0
|
||||
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
||||
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -192,93 +195,56 @@ func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
||||
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "archive.txt")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "old")
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
sharedRoot := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
|
||||
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func pruneSingleOwnerState(now time.Time) state.DistributorState {
|
||||
func pruneCatalogState(now time.Time) state.CatalogState {
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
publishedAt := now.Add(-96 * time.Hour)
|
||||
return state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
createdAt := now.Add(-96 * time.Hour)
|
||||
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
|
||||
return state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: []state.CatalogOutputFile{{
|
||||
Path: "old.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
DistributorVersion: "test",
|
||||
Outputs: []state.OutputFile{{
|
||||
Path: "old.txt",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}, {
|
||||
Path: "fresh.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
CreatedAt: now.Add(-24 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func pruneSharedRootState(now time.Time) state.SharedRootState {
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
archive := state.CurrentOwnerScope("reports", "archive")
|
||||
html := state.CurrentOwnerScope("reports", "html")
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: archive,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
}, {
|
||||
Scope: html,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "archive.txt",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
Owner: archive,
|
||||
SourceID: manifest.ID,
|
||||
SourceDigest: manifest.Digest,
|
||||
SourceCreated: manifest.Created,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
CreatedAt: now.Add(-24 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
}, {
|
||||
Path: "html.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "html",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
Owner: html,
|
||||
SourceID: manifest.ID,
|
||||
SourceDigest: manifest.Digest,
|
||||
SourceCreated: manifest.Created,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}},
|
||||
@@ -319,18 +285,31 @@ func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
|
||||
func writeFakeCatalogState(t *testing.T, backend *fake.Backend, catalog state.CatalogState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
data, err := json.MarshalIndent(catalog, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal single-owner state: %v", err)
|
||||
t.Fatalf("marshal catalog state: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
||||
for _, output := range destinationState.Outputs {
|
||||
for _, output := range catalog.Outputs {
|
||||
testutil.WriteFakeFile(t, backend, output.Path, "managed")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeCatalogState(t *testing.T, backend *fake.Backend) state.CatalogState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
|
||||
t.Helper()
|
||||
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {
|
||||
|
||||
@@ -158,62 +158,25 @@ func buildReconcileStateReport(ctx context.Context, backend storage.Backend, pip
|
||||
DryRun: options.DryRun,
|
||||
}
|
||||
scope := state.CurrentOwnerScope(pipeline.ID, destination.ID)
|
||||
if document.SingleOwner != nil {
|
||||
return reconcileSingleOwnerState(ctx, backend, statePath, *document.SingleOwner, scope, report, options)
|
||||
if document.Catalog != nil {
|
||||
return reconcileCatalogState(ctx, backend, statePath, *document.Catalog, scope, report, options)
|
||||
}
|
||||
return reconcileSharedRootState(ctx, backend, statePath, *document.SharedRoot, scope, report, options)
|
||||
return ReconcileStateReport{}, unsupportedStateDocumentError(document)
|
||||
}
|
||||
|
||||
func reconcileSingleOwnerState(ctx context.Context, backend storage.Backend, statePath string, destinationState state.DistributorState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
if destinationState.PipelineID != scope.PipelineID || destinationState.DestinationID != scope.DestinationID {
|
||||
return ReconcileStateReport{}, fmt.Errorf("state owner is %s/%s, not %s/%s", destinationState.PipelineID, destinationState.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
report.StateSchema = destinationState.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID}
|
||||
managed := state.ManagedOutputPaths(destinationState)
|
||||
missing, err := missingSingleOwnerOutputs(ctx, backend, destinationState.Outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.CheckedCount = len(managed)
|
||||
report.MissingManagedOutputs = missing
|
||||
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.UnmanagedEntries = unmanaged
|
||||
report.WouldChange = options.DryRun && len(missing) > 0
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
next, changed := state.RemoveMissingOutputs(destinationState, missingPaths)
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.Validate(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func reconcileSharedRootState(ctx context.Context, backend storage.Backend, statePath string, sharedRoot state.SharedRootState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
report.StateSchema = sharedRoot.SchemaVersion
|
||||
func reconcileCatalogState(ctx context.Context, backend storage.Backend, statePath string, catalog state.CatalogState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
report.StateSchema = catalog.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{
|
||||
PipelineID: scope.PipelineID,
|
||||
DestinationID: scope.DestinationID,
|
||||
AllOwners: options.AllOwners,
|
||||
}
|
||||
managed := sharedRoot.AllManagedOutputPaths()
|
||||
outputs := sharedRoot.Outputs
|
||||
managed := state.CatalogManagedOutputPaths(catalog)
|
||||
outputs := catalog.Outputs
|
||||
if !options.AllOwners {
|
||||
outputs = sharedRootOutputsForOwner(sharedRoot.Outputs, scope)
|
||||
outputs = state.CatalogOutputsForOwner(catalog.Outputs, scope)
|
||||
}
|
||||
missing, err := missingSharedRootOutputs(ctx, backend, outputs)
|
||||
missing, err := missingCatalogOutputs(ctx, backend, outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
@@ -228,17 +191,17 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
var next state.SharedRootState
|
||||
var next state.CatalogState
|
||||
var changed bool
|
||||
if options.AllOwners {
|
||||
next, changed = state.RemoveMissingSharedRootOutputs(sharedRoot, missingPaths)
|
||||
next, changed = state.RemoveMissingCatalogOutputs(catalog, missingPaths)
|
||||
} else {
|
||||
next, changed = state.RemoveMissingSharedRootOwnerOutputs(sharedRoot, scope, missingPaths)
|
||||
next, changed = state.RemoveMissingCatalogOwnerOutputs(catalog, scope, missingPaths)
|
||||
}
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
if err := state.ValidateCatalog(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
@@ -249,21 +212,7 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func missingSingleOwnerOutputs(ctx context.Context, backend storage.Backend, outputs []state.OutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
if storage.IsNotFound(err) {
|
||||
missing = append(missing, ReconcileStatePath{Path: output.Path, StorageStatus: "missing"})
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outputs []state.SharedRootOutputFile) ([]ReconcileStatePath, error) {
|
||||
func missingCatalogOutputs(ctx context.Context, backend storage.Backend, outputs []state.CatalogOutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
@@ -271,8 +220,8 @@ func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outp
|
||||
missing = append(missing, ReconcileStatePath{
|
||||
Path: output.Path,
|
||||
OwnerScope: &ReconcileStateOwnerScope{
|
||||
PipelineID: output.Owner.PipelineID,
|
||||
DestinationID: output.Owner.DestinationID,
|
||||
PipelineID: output.PipelineID,
|
||||
DestinationID: output.DestinationID,
|
||||
},
|
||||
StorageStatus: "missing",
|
||||
})
|
||||
@@ -320,16 +269,6 @@ func unmanagedEntries(ctx context.Context, backend storage.Backend, managedPaths
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func sharedRootOutputsForOwner(outputs []state.SharedRootOutputFile, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
selected := make([]state.SharedRootOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if output.Owner == scope {
|
||||
selected = append(selected, output)
|
||||
}
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func missingReportPaths(missing []ReconcileStatePath) []string {
|
||||
paths := make([]string, 0, len(missing))
|
||||
for _, item := range missing {
|
||||
|
||||
@@ -2,7 +2,6 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -17,9 +16,9 @@ import (
|
||||
func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
catalog := pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC))
|
||||
writeFakeCatalogState(t, backend, catalog)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed output: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
@@ -35,14 +34,14 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
|
||||
if !report.WouldChange || report.Changed {
|
||||
t.Fatalf("report changed=%t would_change=%t, want dry-run pending change", report.Changed, report.WouldChange)
|
||||
}
|
||||
if got := reportPathList(report.MissingManagedOutputs); got != "summary.txt" {
|
||||
t.Fatalf("missing outputs = %q, want summary.txt", got)
|
||||
if got := reportPathList(report.MissingManagedOutputs); got != "fresh.txt" {
|
||||
t.Fatalf("missing outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
if got := entryPathList(report.UnmanagedEntries); got != "extra.txt" {
|
||||
t.Fatalf("unmanaged entries = %q, want extra.txt", got)
|
||||
}
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
}
|
||||
@@ -50,9 +49,8 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
|
||||
func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed output: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
@@ -67,12 +65,15 @@ func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *t
|
||||
if !report.Changed || report.WouldChange {
|
||||
t.Fatalf("report changed=%t would_change=%t, want applied change", report.Changed, report.WouldChange)
|
||||
}
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if err := state.Validate(destinationState); err != nil {
|
||||
t.Fatalf("Validate() repaired state error = %v", err)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if err := state.ValidateCatalog(repaired); err != nil {
|
||||
t.Fatalf("ValidateCatalog() repaired state error = %v", err)
|
||||
}
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
|
||||
t.Fatalf("state outputs = %q, want report.md", got)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want old.txt,html.txt", got)
|
||||
}
|
||||
if repaired.SchemaVersion != state.CatalogSchemaVersion {
|
||||
t.Fatalf("state schema_version = %d, want %d", repaired.SchemaVersion, state.CatalogSchemaVersion)
|
||||
}
|
||||
testutil.AssertFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
}
|
||||
@@ -99,12 +100,11 @@ func TestReconcileStateInvalidStateFailsWithoutRewrite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
|
||||
func TestReconcileStateOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
sharedRoot := reconcileSharedRootFixture(t)
|
||||
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed outputs: %v", err)
|
||||
}
|
||||
|
||||
@@ -118,18 +118,17 @@ func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T)
|
||||
if !report.Changed {
|
||||
t.Fatal("report changed = false, want true")
|
||||
}
|
||||
repaired := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := strings.Join(repaired.AllManagedOutputPaths(), ","); got != "report.html" {
|
||||
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
func TestReconcileStateAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
sharedRoot := reconcileSharedRootFixture(t)
|
||||
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed outputs: %v", err)
|
||||
}
|
||||
|
||||
@@ -141,12 +140,12 @@ func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.Changed || report.CheckedCount != 2 {
|
||||
if !report.Changed || report.CheckedCount != 3 {
|
||||
t.Fatalf("report changed=%t checked=%d, want all-owner repair", report.Changed, report.CheckedCount)
|
||||
}
|
||||
repaired := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := repaired.AllManagedOutputPaths(); len(got) != 0 {
|
||||
t.Fatalf("shared-root outputs = %#v, want none", got)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -165,93 +164,6 @@ func reconcileStateS3Config(t *testing.T) config.Config {
|
||||
return cfg
|
||||
}
|
||||
|
||||
func readFakeSingleOwnerState(t *testing.T, backend *fake.Backend) state.DistributorState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func writeFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend, sharedRoot state.SharedRootState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(sharedRoot, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared-root state: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
testutil.WriteFakeFile(t, backend, output.Path, "old")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend) state.SharedRootState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
}
|
||||
sharedRoot, err := state.ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
}
|
||||
return sharedRoot
|
||||
}
|
||||
|
||||
func reconcileSharedRootFixture(t *testing.T) state.SharedRootState {
|
||||
t.Helper()
|
||||
source := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
htmlSource := source
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: state.CurrentOwnerScope("reports", "archive"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: source},
|
||||
}, {
|
||||
Scope: state.CurrentOwnerScope("reports", "html"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
||||
Source: state.SourceState{Manifest: htmlSource},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
Owner: state.CurrentOwnerScope("reports", "archive"),
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
Kind: state.OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
SHA256: "sha256:" + strings.Repeat("a", 64),
|
||||
Size: 128,
|
||||
Owner: state.CurrentOwnerScope("reports", "html"),
|
||||
SourceID: htmlSource.ID,
|
||||
SourceDigest: htmlSource.Digest,
|
||||
SourceCreated: htmlSource.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func reportPathList(paths []ReconcileStatePath) string {
|
||||
values := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
|
||||
@@ -69,10 +69,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
|
||||
Publish: *request.destination.Publish,
|
||||
Transform: request.destination.Transform,
|
||||
Links: request.destination.Links,
|
||||
State: request.destination.State,
|
||||
Reconciliation: request.destination.Reconciliation,
|
||||
Workflow: request.destination.Workflow,
|
||||
Transformers: request.transforms,
|
||||
Transfer: request.destination.Transfer,
|
||||
DistributorVersion: Version,
|
||||
Force: request.options.Force,
|
||||
}
|
||||
@@ -82,8 +80,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
|
||||
}
|
||||
if isFixedPathDestination(request.destination) {
|
||||
plan.PathMapping = config.PathMappingFixed
|
||||
if request.options.DryRun && isDestructiveFixedPathAction(plan.Action) {
|
||||
warning := fixedPathReplacementWarning(plan)
|
||||
if request.options.DryRun && isFixedPathWorkflowAction(plan.Action) {
|
||||
warning := fixedPathWorkflowWarning(plan)
|
||||
request.recorder.addPipelineWarning(request.pipelineIndex, warning)
|
||||
}
|
||||
}
|
||||
@@ -141,6 +139,7 @@ func (recorder *runReportRecorder) recordDestinationFailure(pipelineIndex int, f
|
||||
recorder.failures.add(failure.pipelineID, failure.destinationID, failure.backend, storage.DisplayPath(failure.bundlePath), failure.err)
|
||||
recorder.summary.recordFailure()
|
||||
if includeAction {
|
||||
recorder.summary.recordFailureAction(action.Action)
|
||||
recorder.addPipelineAction(pipelineIndex, action)
|
||||
}
|
||||
}
|
||||
@@ -161,5 +160,8 @@ func completePlanIdentity(plan publish.Plan, pipeline config.Pipeline, destinati
|
||||
if plan.DestinationBundlePath == "" {
|
||||
plan.DestinationBundlePath = selection.DestinationBundlePath
|
||||
}
|
||||
if plan.Workflow == "" {
|
||||
plan.Workflow = destination.Workflow
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
@@ -6,20 +6,19 @@ import (
|
||||
)
|
||||
|
||||
func shouldNotify(action publish.Action) bool {
|
||||
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
||||
return action == publish.ActionPublishNew || action == publish.ActionUpsertAdditive || action == publish.ActionReplaceCatalog || action == publish.ActionForceReplace
|
||||
}
|
||||
|
||||
func notifyEvent(plan publish.Plan) notify.Event {
|
||||
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
||||
for _, output := range plan.Outputs {
|
||||
stateOutput := output.StateOutputFile()
|
||||
outputs = append(outputs, notify.Output{
|
||||
Path: stateOutput.Path,
|
||||
Kind: stateOutput.Kind,
|
||||
SourcePath: stateOutput.SourcePath,
|
||||
Transform: stateOutput.Transform,
|
||||
SHA256: stateOutput.SHA256,
|
||||
Size: stateOutput.Size,
|
||||
Path: output.DestinationPath,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
})
|
||||
}
|
||||
return notify.Event{
|
||||
|
||||
@@ -60,7 +60,7 @@ func writeRunActionLine(w io.Writer, action RunActionRecord) {
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", action.BundlePath, destinationID, action.Backend, pathMappingRecordSummary(action), action.Reason)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, outputRecordSummary(action.Outputs), action.Reason)
|
||||
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, workflowRecordSummary(action), outputRecordSummary(action.Outputs), action.Reason)
|
||||
}
|
||||
|
||||
func pathMappingRecordSummary(action RunActionRecord) string {
|
||||
@@ -70,6 +70,13 @@ func pathMappingRecordSummary(action RunActionRecord) string {
|
||||
return fmt.Sprintf(" path_mapping=fixed target=%s", action.DestinationPath)
|
||||
}
|
||||
|
||||
func workflowRecordSummary(action RunActionRecord) string {
|
||||
if action.Workflow == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" workflow=%s", action.Workflow)
|
||||
}
|
||||
|
||||
func outputRecordSummary(outputs []RunOutputRecord) string {
|
||||
if len(outputs) == 0 {
|
||||
return "none"
|
||||
@@ -136,6 +143,7 @@ type RunActionRecord struct {
|
||||
BundlePath string `json:"bundle_path"`
|
||||
DestinationPath string `json:"destination_path"`
|
||||
PathMapping string `json:"path_mapping,omitempty"`
|
||||
Workflow string `json:"workflow,omitempty"`
|
||||
Action string `json:"action"`
|
||||
PrimaryURL string `json:"primary_url,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
@@ -158,6 +166,13 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
|
||||
if destinationID == "" {
|
||||
destinationID = "unknown"
|
||||
}
|
||||
action := "error"
|
||||
outputs := []RunOutputRecord{}
|
||||
switch plan.Action {
|
||||
case publish.ActionFailUnmanaged, publish.ActionFailConflict:
|
||||
action = string(plan.Action)
|
||||
outputs = runOutputsFromPlan(plan.Outputs)
|
||||
}
|
||||
return RunActionRecord{
|
||||
PipelineID: plan.PipelineID,
|
||||
DestinationID: destinationID,
|
||||
@@ -166,10 +181,11 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
|
||||
BundlePath: storage.DisplayPath(plan.BundlePath),
|
||||
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
||||
PathMapping: plan.PathMapping,
|
||||
Action: "error",
|
||||
Workflow: plan.Workflow,
|
||||
Action: action,
|
||||
PrimaryURL: plan.PrimaryURL,
|
||||
Reason: planErr.Error(),
|
||||
Outputs: []RunOutputRecord{},
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
return RunActionRecord{
|
||||
@@ -180,6 +196,7 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
|
||||
BundlePath: storage.DisplayPath(plan.BundlePath),
|
||||
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
|
||||
PathMapping: plan.PathMapping,
|
||||
Workflow: plan.Workflow,
|
||||
Action: string(plan.Action),
|
||||
PrimaryURL: plan.PrimaryURL,
|
||||
Reason: plan.Reason,
|
||||
@@ -203,15 +220,14 @@ func errorAction(pipelineID, destinationID, backend, bundlePath string, err erro
|
||||
func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
|
||||
results := make([]RunOutputRecord, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
stateOutput := output.StateOutputFile()
|
||||
results = append(results, RunOutputRecord{
|
||||
Path: stateOutput.Path,
|
||||
Kind: stateOutput.Kind,
|
||||
SourcePath: stateOutput.SourcePath,
|
||||
Transform: stateOutput.Transform,
|
||||
URL: stateOutput.URL,
|
||||
SHA256: stateOutput.SHA256,
|
||||
Size: stateOutput.Size,
|
||||
Path: output.DestinationPath,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
})
|
||||
}
|
||||
return results
|
||||
|
||||
@@ -62,12 +62,22 @@ func fixedPathSelectionWarning(pipelineID, destinationID string, selections []de
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
|
||||
}
|
||||
|
||||
func isDestructiveFixedPathAction(action publish.Action) bool {
|
||||
return action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
|
||||
func isFixedPathWorkflowAction(action publish.Action) bool {
|
||||
return action == publish.ActionUpsertAdditive || action == publish.ActionReplaceCatalog || action == publish.ActionForceReplace
|
||||
}
|
||||
|
||||
func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
func fixedPathWorkflowWarning(plan publish.Plan) OutputWarning {
|
||||
switch plan.Action {
|
||||
case publish.ActionReplaceCatalog:
|
||||
if plan.ClearDestinationRoot {
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s clears destination root before writing selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
}
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s replaces current-owner catalog outputs for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
case publish.ActionUpsertAdditive:
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s upserts planned outputs at destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
default:
|
||||
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s writes selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
|
||||
}
|
||||
}
|
||||
|
||||
func destinationIDs(destinations []config.Destination) []string {
|
||||
|
||||
@@ -10,9 +10,12 @@ type runSummary struct {
|
||||
dryRun bool
|
||||
planned int
|
||||
publishNew int
|
||||
replaceOlder int
|
||||
upsertAdditive int
|
||||
replaceCatalog int
|
||||
skipSame int
|
||||
forceReplace int
|
||||
skipped int
|
||||
failUnmanaged int
|
||||
failConflict int
|
||||
failures int
|
||||
fixedPath int
|
||||
}
|
||||
@@ -22,12 +25,23 @@ func (s *runSummary) recordPlan(action publish.Action) {
|
||||
switch action {
|
||||
case publish.ActionPublishNew:
|
||||
s.publishNew++
|
||||
case publish.ActionReplaceOlder:
|
||||
s.replaceOlder++
|
||||
case publish.ActionUpsertAdditive:
|
||||
s.upsertAdditive++
|
||||
case publish.ActionReplaceCatalog:
|
||||
s.replaceCatalog++
|
||||
case publish.ActionForceReplace:
|
||||
s.forceReplace++
|
||||
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
||||
s.skipped++
|
||||
case publish.ActionSkipSame:
|
||||
s.skipSame++
|
||||
}
|
||||
}
|
||||
|
||||
func (s *runSummary) recordFailureAction(action string) {
|
||||
switch action {
|
||||
case string(publish.ActionFailUnmanaged):
|
||||
s.failUnmanaged++
|
||||
case string(publish.ActionFailConflict):
|
||||
s.failConflict++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,16 +57,19 @@ type RunSummaryCounters struct {
|
||||
Status string `json:"status"`
|
||||
Planned int `json:"planned"`
|
||||
PublishNew int `json:"publish_new"`
|
||||
ReplaceOlder int `json:"replace_older"`
|
||||
UpsertAdditive int `json:"upsert_additive"`
|
||||
ReplaceCatalog int `json:"replace_catalog"`
|
||||
SkipSame int `json:"skip_same"`
|
||||
ForceReplace int `json:"force_replace"`
|
||||
Skipped int `json:"skipped"`
|
||||
FailUnmanaged int `json:"fail_unmanaged"`
|
||||
FailConflict int `json:"fail_conflict"`
|
||||
Failed int `json:"failed"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
FixedPath int `json:"fixed_path"`
|
||||
}
|
||||
|
||||
func (s RunSummaryCounters) Line() string {
|
||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.ReplaceOlder, s.ForceReplace, s.Skipped, s.Failed, s.DryRun, s.FixedPath)
|
||||
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d upsert_additive=%d replace_catalog=%d skip_same=%d force_replace=%d fail_unmanaged=%d fail_conflict=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.UpsertAdditive, s.ReplaceCatalog, s.SkipSame, s.ForceReplace, s.FailUnmanaged, s.FailConflict, s.Failed, s.DryRun, s.FixedPath)
|
||||
}
|
||||
|
||||
func (s runSummary) Result() RunSummaryCounters {
|
||||
@@ -64,9 +81,12 @@ func (s runSummary) Result() RunSummaryCounters {
|
||||
Status: status,
|
||||
Planned: s.planned,
|
||||
PublishNew: s.publishNew,
|
||||
ReplaceOlder: s.replaceOlder,
|
||||
UpsertAdditive: s.upsertAdditive,
|
||||
ReplaceCatalog: s.replaceCatalog,
|
||||
SkipSame: s.skipSame,
|
||||
ForceReplace: s.forceReplace,
|
||||
Skipped: s.skipped,
|
||||
FailUnmanaged: s.failUnmanaged,
|
||||
FailConflict: s.failConflict,
|
||||
Failed: s.failures,
|
||||
DryRun: s.dryRun,
|
||||
FixedPath: s.fixedPath,
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
@@ -42,8 +41,8 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"Configured pipelines: 1",
|
||||
"- pipeline=reports source=local bundles=1 destinations=archive",
|
||||
"bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt",
|
||||
"Final status: ok planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true",
|
||||
"bundle=. destination=archive backend=local action=publish_new workflow=additive outputs=report.md,summary.txt",
|
||||
"Final status: ok planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("Run() output = %q, want substring %q", output, want)
|
||||
@@ -211,75 +210,14 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
||||
if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
|
||||
t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
|
||||
}
|
||||
if destinationState.Source.Manifest.ID != manifest.ID {
|
||||
t.Fatalf("state source id = %q, want %q", destinationState.Source.Manifest.ID, manifest.ID)
|
||||
if destinationState.SourceID != manifest.ID {
|
||||
t.Fatalf("state source id = %q, want %q", destinationState.SourceID, manifest.ID)
|
||||
}
|
||||
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if destinationState.Links != nil || destinationState.Outputs[0].URL != "" {
|
||||
t.Fatalf("state links = %#v output URL=%q, want absent", destinationState.Links, destinationState.Outputs[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSharedRootDryRunWritesNoOutputsOrState(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||
Files: []testFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
configPath := writeSharedRootLocalConfig(t, sourceRoot, destinationRoot)
|
||||
|
||||
cfg, err := config.LoadFile(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment)
|
||||
if err != nil {
|
||||
t.Fatalf("dry-run buildRunReportWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if got, want := report.Actions[0].Action, string(publish.ActionPublishNew); got != want {
|
||||
t.Fatalf("dry-run action = %q, want %q", got, want)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("output stat error = %v, want absent", statErr)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("state file stat error = %v, want absent", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPublishesTwoPipelinesIntoSharedRoot(t *testing.T) {
|
||||
firstSourceRoot := t.TempDir()
|
||||
secondSourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, firstSourceRoot, "", testBundleOptions{
|
||||
ID: "reports.first",
|
||||
Files: []testFile{{Path: "first.md", Data: "# First\n"}},
|
||||
})
|
||||
writeSourceBundle(t, secondSourceRoot, "", testBundleOptions{
|
||||
ID: "reports.second",
|
||||
Files: []testFile{{Path: "second.md", Data: "# Second\n"}},
|
||||
})
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeTwoPipelineSharedRootConfig(t, firstSourceRoot, secondSourceRoot, destinationRoot)})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "first.md"), "# First\n")
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "second.md"), "# Second\n")
|
||||
destinationState := readSharedRootStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got, want := len(destinationState.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-first", "archive")); !ok {
|
||||
t.Fatal("reports-first/archive owner missing")
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-second", "archive")); !ok {
|
||||
t.Fatal("reports-second/archive owner missing")
|
||||
}
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "first.md,second.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
if destinationState.PrimaryURL != "" || destinationState.Outputs[0].URL != "" {
|
||||
t.Fatalf("state primary URL = %q output URL=%q, want absent", destinationState.PrimaryURL, destinationState.Outputs[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,8 +362,8 @@ func TestRunRecordsLinksForNestedBundlePath(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName))
|
||||
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||
t.Fatalf("state links = %#v, want source primary URL", destinationState.Links)
|
||||
if destinationState.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||
t.Fatalf("state primary URL = %q, want source primary URL", destinationState.PrimaryURL)
|
||||
}
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||
@@ -449,8 +387,8 @@ func TestRunRecordsLinksForFixedIndexDestination(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/latest/" {
|
||||
t.Fatalf("state links = %#v, want fixed index primary URL", destinationState.Links)
|
||||
if destinationState.PrimaryURL != "https://reports.example.com/latest/" {
|
||||
t.Fatalf("state primary URL = %q, want fixed index primary URL", destinationState.PrimaryURL)
|
||||
}
|
||||
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
@@ -489,8 +427,8 @@ func TestRunFixedPathPublishesNewestBundleAtDestinationRoot(t *testing.T) {
|
||||
t.Fatalf("nested new report stat error = %v, want not exist", err)
|
||||
}
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if destinationState.Source.Manifest.ID != "reports.new" {
|
||||
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
|
||||
if destinationState.SourceID != "reports.new" {
|
||||
t.Fatalf("state source id = %q, want reports.new", destinationState.SourceID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -519,8 +457,8 @@ func TestRunFixedPathTieBreaksByBundlePath(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if destinationState.Source.Manifest.ID != "reports.a" {
|
||||
t.Fatalf("state source id = %q, want reports.a", destinationState.Source.Manifest.ID)
|
||||
if destinationState.SourceID != "reports.a" {
|
||||
t.Fatalf("state source id = %q, want reports.a", destinationState.SourceID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -568,7 +506,7 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
|
||||
{Path: "summary.txt", Data: "Old summary\n"},
|
||||
},
|
||||
})
|
||||
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
|
||||
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingFixed, config.WorkflowReplacement)
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
@@ -592,8 +530,9 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, want := range []string{
|
||||
"Warning: pipeline=reports destination=archive path_mapping=fixed action=replace_older replaces destination root for selected_bundle=new",
|
||||
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_older",
|
||||
"Warning: pipeline=reports destination=archive path_mapping=fixed workflow=replacement action=replace_catalog replaces current-owner catalog outputs for selected_bundle=new",
|
||||
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_catalog workflow=replacement outputs=report.md,summary.txt reason=\"\"",
|
||||
"replace_catalog=1",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||
@@ -602,7 +541,7 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
|
||||
}
|
||||
|
||||
func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
|
||||
func TestRunJSONIncludesWorkflowActionAndSummary(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
|
||||
@@ -613,12 +552,10 @@ func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
|
||||
{Path: "summary.txt", Data: "Old summary\n"},
|
||||
},
|
||||
})
|
||||
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
|
||||
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingFixed, config.WorkflowReplacement)
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
|
||||
|
||||
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
|
||||
ID: "reports.new",
|
||||
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||
@@ -628,44 +565,35 @@ func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("second Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if destinationState.Source.Manifest.ID != "reports.new" {
|
||||
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFixedPathSkipsWhenDestinationStateIsNewer(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
newer := testutil.ValidManifest(testutil.BundleOptions{
|
||||
ID: "reports.newer",
|
||||
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||
})
|
||||
writeDestinationState(t, destinationRoot, "", newer)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nExisting.\n"), 0o600); err != nil {
|
||||
t.Fatalf("write existing report: %v", err)
|
||||
}
|
||||
writeSourceBundle(t, sourceRoot, "older", testBundleOptions{
|
||||
ID: "reports.older",
|
||||
Created: testutil.DefaultCreated,
|
||||
})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
|
||||
ConfigPath: configPath,
|
||||
DryRun: true,
|
||||
Stdout: &stdout,
|
||||
OutputFormat: OutputFormatJSON,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
|
||||
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
|
||||
result := decodeAppResult(t, stdout.String())
|
||||
actions, ok := result["actions"].([]any)
|
||||
if !ok || len(actions) != 1 {
|
||||
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||
}
|
||||
action, ok := actions[0].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("action = %#v, want object", actions[0])
|
||||
}
|
||||
if action["action"] != "replace_catalog" || action["workflow"] != "replacement" || action["reason"] != nil {
|
||||
t.Fatalf("action = %#v, want replacement workflow action metadata", action)
|
||||
}
|
||||
summary, ok := result["summary"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("summary = %#v, want object", result["summary"])
|
||||
}
|
||||
if summary["replace_catalog"] != float64(1) || summary["upsert_additive"] != float64(0) || summary["force_replace"] != float64(0) {
|
||||
t.Fatalf("summary = %#v, want replacement workflow counter only", summary)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nExisting.\n")
|
||||
}
|
||||
|
||||
func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
|
||||
@@ -709,6 +637,10 @@ func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("unmanaged stat error = %v, want removed", err)
|
||||
}
|
||||
catalog := readLocalCatalogState(t, destinationRoot)
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
|
||||
@@ -721,14 +653,6 @@ func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
|
||||
{Path: "summary.txt", Data: "Old summary\n"},
|
||||
},
|
||||
})
|
||||
writeSourceBundle(t, localSourceRoot, "new", testBundleOptions{
|
||||
ID: "reports.new",
|
||||
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||
Files: []testFile{
|
||||
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||
{Path: "summary.txt", Data: "New summary\n"},
|
||||
},
|
||||
})
|
||||
s3Destination := fake.New()
|
||||
sshDestination := fake.New()
|
||||
cfg := config.Config{Pipelines: []config.Pipeline{{
|
||||
@@ -760,6 +684,30 @@ func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
|
||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{}, provider); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nOld.\n")
|
||||
testutil.AssertFakeFile(t, sshDestination, "summary.txt", "Old summary\n")
|
||||
|
||||
writeSourceBundle(t, localSourceRoot, "new", testBundleOptions{
|
||||
ID: "reports.new",
|
||||
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||
Files: []testFile{
|
||||
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||
{Path: "summary.txt", Data: "New summary\n"},
|
||||
},
|
||||
})
|
||||
var stdout bytes.Buffer
|
||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &stdout}, provider); err != nil {
|
||||
t.Fatalf("second Run() error = %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"destination=object-latest backend=s3 path_mapping=fixed target=. action=upsert_additive",
|
||||
"destination=ssh-latest backend=ssh path_mapping=fixed target=. action=upsert_additive",
|
||||
"planned=2",
|
||||
} {
|
||||
if !strings.Contains(stdout.String(), want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
|
||||
}
|
||||
}
|
||||
testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeFile(t, s3Destination, "summary.txt", "New summary\n")
|
||||
testutil.AssertFakeMissing(t, s3Destination, "new/report.md")
|
||||
@@ -828,17 +776,22 @@ func TestRunNotifiesGeneratedOutputMetadata(t *testing.T) {
|
||||
func TestRunNotifiesAfterReplacement(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
older := manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
writeDestinationState(t, destinationRoot, "", older)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old output: %v", err)
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, config.WorkflowReplacement)
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||
Files: []testFile{
|
||||
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||
{Path: "summary.txt", Data: "New summary\n"},
|
||||
},
|
||||
})
|
||||
notifier := &recordingNotifier{}
|
||||
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||
ConfigPath: configPath,
|
||||
Notifier: notifier,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -847,55 +800,8 @@ func TestRunNotifiesAfterReplacement(t *testing.T) {
|
||||
if got, want := len(notifier.events), 1; got != want {
|
||||
t.Fatalf("notification count = %d, want %d", got, want)
|
||||
}
|
||||
if notifier.events[0].Action != "replace_older" {
|
||||
t.Fatalf("notification action = %q, want replace_older", notifier.events[0].Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMergeReconciliationRetainsManagedOutput(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
older := manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
defaultManifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
older.Files = append([]bundle.ManifestFile(nil), defaultManifest.Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
writeDestinationState(t, destinationRoot, "", older)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old report: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "summary.txt"), []byte("old summary\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old summary: %v", err)
|
||||
}
|
||||
configPath := writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
reconciliation:
|
||||
mode: merge
|
||||
`)
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "old summary\n")
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
if notifier.events[0].Action != "replace_catalog" {
|
||||
t.Fatalf("notification action = %q, want replace_catalog", notifier.events[0].Action)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1002,8 +908,8 @@ func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged planned file: %v", err)
|
||||
}
|
||||
cfg, err := config.LoadFile(writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination))
|
||||
if err != nil {
|
||||
@@ -1020,8 +926,8 @@ func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
|
||||
if got, want := len(report.Actions), 2; got != want {
|
||||
t.Fatalf("action count = %d, want %d", got, want)
|
||||
}
|
||||
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "error" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
|
||||
t.Fatalf("first action = %#v, want archive-one error", report.Actions[0])
|
||||
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "fail_unmanaged" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
|
||||
t.Fatalf("first action = %#v, want archive-one unmanaged failure", report.Actions[0])
|
||||
}
|
||||
if report.Actions[1].DestinationID != "archive-two" || report.Actions[1].Action != "publish_new" {
|
||||
t.Fatalf("second action = %#v, want archive-two publish_new", report.Actions[1])
|
||||
@@ -1156,7 +1062,7 @@ func TestRunStillRunsAllConfiguredPipelines(t *testing.T) {
|
||||
testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
|
||||
}
|
||||
|
||||
func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
|
||||
func TestRunNotifiesForAdditiveUpsert(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
@@ -1164,14 +1070,24 @@ func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||
Files: []testFile{
|
||||
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||
{Path: "summary.txt", Data: "New summary\n"},
|
||||
},
|
||||
})
|
||||
notifier := &recordingNotifier{}
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Notifier: notifier})
|
||||
if err != nil {
|
||||
t.Fatalf("second Run() error = %v", err)
|
||||
}
|
||||
if len(notifier.events) != 0 {
|
||||
t.Fatalf("notifications = %#v, want none", notifier.events)
|
||||
if got, want := len(notifier.events), 1; got != want {
|
||||
t.Fatalf("notification count = %d, want %d", got, want)
|
||||
}
|
||||
if notifier.events[0].Action != "upsert_additive" {
|
||||
t.Fatalf("notification action = %q, want upsert_additive", notifier.events[0].Action)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1202,8 +1118,8 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged planned file: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
@@ -1219,9 +1135,9 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
|
||||
}
|
||||
output := stdout.String()
|
||||
for _, want := range []string{
|
||||
"destination=archive-one backend=local action=error",
|
||||
"destination=archive-two backend=local action=publish_new",
|
||||
"Final status: failed planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=1 dry_run=false",
|
||||
"destination=archive-one backend=local action=fail_unmanaged workflow=additive",
|
||||
"destination=archive-two backend=local action=publish_new workflow=additive",
|
||||
"Final status: failed planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=1 fail_conflict=0 failed=1 dry_run=false",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||
@@ -1471,6 +1387,55 @@ func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
statePath := filepath.Join(destinationRoot, storage.StateFileName)
|
||||
stateBefore, err := os.ReadFile(statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read state before second run: %v", err)
|
||||
}
|
||||
reportBefore, err := os.ReadFile(filepath.Join(destinationRoot, "report.md"))
|
||||
if err != nil {
|
||||
t.Fatalf("read report before second run: %v", err)
|
||||
}
|
||||
notifier := &recordingNotifier{}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err = Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout, Notifier: notifier})
|
||||
if err != nil {
|
||||
t.Fatalf("second Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=skip_same") {
|
||||
t.Fatalf("stdout = %q, want skip_same", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Final status: ok planned=1 publish_new=0 upsert_additive=0 replace_catalog=0 skip_same=1") {
|
||||
t.Fatalf("stdout = %q, want skip_same summary", stdout.String())
|
||||
}
|
||||
stateAfter, err := os.ReadFile(statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read state after second run: %v", err)
|
||||
}
|
||||
if string(stateAfter) != string(stateBefore) {
|
||||
t.Fatalf("state changed during skip")
|
||||
}
|
||||
reportAfter, err := os.ReadFile(filepath.Join(destinationRoot, "report.md"))
|
||||
if err != nil {
|
||||
t.Fatalf("read report after second run: %v", err)
|
||||
}
|
||||
if string(reportAfter) != string(reportBefore) {
|
||||
t.Fatalf("report changed during skip")
|
||||
}
|
||||
if got, want := len(notifier.events), 0; got != want {
|
||||
t.Fatalf("notification count = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplacementSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, config.WorkflowReplacement)
|
||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout})
|
||||
@@ -1482,63 +1447,6 @@ func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReplacesOlderDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
older := manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
writeDestinationState(t, destinationRoot, "", older)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old output: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=replace_older") {
|
||||
t.Fatalf("stdout = %q, want replace_older", stdout.String())
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
}
|
||||
|
||||
func TestRunSkipsNewerDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
newer := manifest
|
||||
newer.Created = newer.Created.Add(time.Hour)
|
||||
writeDestinationState(t, destinationRoot, "", newer)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
|
||||
t.Fatalf("write newer output: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
|
||||
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
|
||||
}
|
||||
|
||||
func TestRunFailsOnConflict(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
manifest.ID = "other.source"
|
||||
writeDestinationState(t, destinationRoot, "", manifest)
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot)})
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
|
||||
t.Fatalf("Run() error = %v, want fail_conflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFailsOnUnmanagedDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -1573,10 +1481,50 @@ func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
|
||||
if !strings.Contains(stdout.String(), "action=force_replace") {
|
||||
t.Fatalf("stdout = %q, want force_replace", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Final status: ok planned=1 publish_new=0 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=1 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=false") {
|
||||
t.Fatalf("stdout = %q, want force_replace counter only", stdout.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("unmanaged file stat error = %v, want not exist", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
catalog := readLocalCatalogState(t, destinationRoot)
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunForceReplacementJSONOutput(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := Run(context.Background(), RunOptions{
|
||||
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||
Force: true,
|
||||
Stdout: &stdout,
|
||||
OutputFormat: OutputFormatJSON,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
result := decodeAppResult(t, stdout.String())
|
||||
actions, ok := result["actions"].([]any)
|
||||
if !ok || len(actions) != 1 {
|
||||
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||
}
|
||||
action, ok := actions[0].(map[string]any)
|
||||
if !ok || action["action"] != "force_replace" || action["workflow"] != "additive" {
|
||||
t.Fatalf("action = %#v, want force_replace additive", actions[0])
|
||||
}
|
||||
summary, ok := result["summary"].(map[string]any)
|
||||
if !ok || summary["force_replace"] != float64(1) || summary["publish_new"] != float64(0) {
|
||||
t.Fatalf("summary = %#v, want force_replace only", result["summary"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFansOutToLocalDestinations(t *testing.T) {
|
||||
@@ -1649,7 +1597,7 @@ func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
|
||||
"pipeline=local-to-ssh source=local",
|
||||
"destination=ssh-archive backend=ssh action=publish_new",
|
||||
"pipeline=ssh-to-local source=ssh",
|
||||
"Final status: ok planned=4 publish_new=4 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true",
|
||||
"Final status: ok planned=4 publish_new=4 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(dryRunOutput.String(), want) {
|
||||
t.Fatalf("dry-run output = %q, want substring %q", dryRunOutput.String(), want)
|
||||
@@ -1722,6 +1670,12 @@ func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
|
||||
testutil.AssertFakeFile(t, sshDestination, "bundle/report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeMissing(t, sshDestination, "bundle/old.txt")
|
||||
testutil.AssertFakeFile(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
||||
for name, backend := range map[string]*fake.Backend{"s3": s3Destination, "ssh": sshDestination} {
|
||||
catalog := readFakeCatalogStateAt(t, backend, "bundle")
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||
t.Fatalf("%s catalog identity = schema %d mode %s", name, catalog.SchemaVersion, catalog.State.Mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDryRunDoesNotWrite(t *testing.T) {
|
||||
@@ -1779,7 +1733,7 @@ func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
||||
return testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||
}
|
||||
|
||||
func writeSharedRootLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
||||
func writeLocalConfigWithWorkflow(t *testing.T, sourceRoot, destinationRoot, pathMapping, workflow string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
@@ -1791,35 +1745,9 @@ pipelines:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
`)
|
||||
}
|
||||
|
||||
func writeTwoPipelineSharedRootConfig(t *testing.T, firstSourceRoot, secondSourceRoot, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports-first
|
||||
source:
|
||||
backend: local
|
||||
path: `+firstSourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
- id: reports-second
|
||||
source:
|
||||
backend: local
|
||||
path: `+secondSourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
workflow: `+workflow+`
|
||||
path_mapping:
|
||||
mode: `+pathMapping+`
|
||||
`)
|
||||
}
|
||||
|
||||
@@ -1879,11 +1807,6 @@ func writeConfigFile(t *testing.T, body string) string {
|
||||
return path
|
||||
}
|
||||
|
||||
func writeDestinationState(t *testing.T, root, relative string, manifest bundle.Manifest) {
|
||||
t.Helper()
|
||||
testutil.WriteDestinationState(t, root, relative, manifest, testutil.DestinationStateOptions{})
|
||||
}
|
||||
|
||||
func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
@@ -1898,26 +1821,93 @@ func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
||||
}
|
||||
}
|
||||
|
||||
func readStateFile(t *testing.T, path string) state.DistributorState {
|
||||
t.Helper()
|
||||
return testutil.ReadDestinationState(t, path)
|
||||
type testDestinationState struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
SourceID string
|
||||
SourceDigest string
|
||||
SourceCreated time.Time
|
||||
PrimaryURL string
|
||||
Outputs []testStateOutput
|
||||
}
|
||||
|
||||
func readSharedRootStateFile(t *testing.T, path string) state.SharedRootState {
|
||||
type testStateOutput struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
func readStateFile(t *testing.T, path string) testDestinationState {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
t.Fatalf("read destination state: %v", err)
|
||||
}
|
||||
destinationState, err := state.ParseSharedRoot(data)
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
view := testDestinationState{Outputs: make([]testStateOutput, 0, len(catalog.Outputs))}
|
||||
for index, output := range catalog.Outputs {
|
||||
if index == 0 {
|
||||
view.PipelineID = output.PipelineID
|
||||
view.DestinationID = output.DestinationID
|
||||
view.SourceID = output.Source.ID
|
||||
view.SourceDigest = output.Source.Digest
|
||||
view.SourceCreated = output.Source.Created
|
||||
if output.URL != "" {
|
||||
view.PrimaryURL = output.URL
|
||||
}
|
||||
}
|
||||
view.Outputs = append(view.Outputs, testStateOutput{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
})
|
||||
}
|
||||
return view
|
||||
}
|
||||
|
||||
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
|
||||
byPath := make(map[string]state.OutputFile, len(outputs))
|
||||
func readLocalCatalogState(t *testing.T, destinationRoot string) state.CatalogState {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if err != nil {
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
func readFakeCatalogStateAt(t *testing.T, backend *fake.Backend, bundlePath string) state.CatalogState {
|
||||
t.Helper()
|
||||
statePath, err := storage.StatePath(bundlePath)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
data, err := backend.ReadFile(context.Background(), statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
func outputsByPath(outputs []testStateOutput) map[string]testStateOutput {
|
||||
byPath := make(map[string]testStateOutput, len(outputs))
|
||||
for _, output := range outputs {
|
||||
byPath[output.Path] = output
|
||||
}
|
||||
|
||||
@@ -3,11 +3,14 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
@@ -92,9 +95,10 @@ func TestExecutePruneDryRunReportsWithoutWriting(t *testing.T) {
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "html.txt"), "other")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,summary.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
@@ -159,13 +163,14 @@ func TestExecutePruneApplyDeletesManagedOutputs(t *testing.T) {
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "summary.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("summary.txt stat error = %v, want not exist", err)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "html.txt"), "other")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
|
||||
t.Fatalf("state file stat error = %v", err)
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := state.ManagedOutputPaths(destinationState); len(got) != 0 {
|
||||
t.Fatalf("state outputs = %#v, want none", got)
|
||||
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "html.txt" {
|
||||
t.Fatalf("state outputs = %q, want html.txt", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
@@ -177,13 +182,16 @@ func writePruneLocalFixture(t *testing.T) (string, string) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
|
||||
writeCatalogDestinationState(t, destinationRoot, manifest, true)
|
||||
for _, file := range testutil.DefaultSourceFiles() {
|
||||
path := filepath.Join(destinationRoot, filepath.FromSlash(file.Path))
|
||||
if err := os.WriteFile(path, []byte(file.Data), 0o600); err != nil {
|
||||
t.Fatalf("write destination output: %v", err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "html.txt"), []byte("other"), 0o600); err != nil {
|
||||
t.Fatalf("write other owner output: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged output: %v", err)
|
||||
}
|
||||
@@ -208,3 +216,71 @@ pipelines:
|
||||
}
|
||||
return destinationRoot, configPath
|
||||
}
|
||||
|
||||
func writeCatalogDestinationState(t *testing.T, root string, manifest bundle.Manifest, includeOtherOwner bool) {
|
||||
t.Helper()
|
||||
createdAt := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
|
||||
outputs := []state.CatalogOutputFile{{
|
||||
Path: "report.md",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "summary.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}}
|
||||
if includeOtherOwner {
|
||||
outputs = append(outputs, state.CatalogOutputFile{
|
||||
Path: "html.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "html",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
})
|
||||
}
|
||||
catalog := state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: outputs,
|
||||
}
|
||||
data, err := json.MarshalIndent(catalog, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal catalog state: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, storage.StateFileName), append(data, '\n'), 0o600); err != nil {
|
||||
t.Fatalf("write catalog state: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readLocalCatalogState(t *testing.T, path string) state.CatalogState {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
@@ -36,9 +36,9 @@ func TestExecuteReconcileStateAppliesByDefault(t *testing.T) {
|
||||
if !strings.Contains(stdout.String(), "status=changed") {
|
||||
t.Fatalf("stdout = %q, want changed status", stdout.String())
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
|
||||
t.Fatalf("state outputs = %q, want report.md", got)
|
||||
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want report.md,html.txt", got)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
if stderr.Len() != 0 {
|
||||
@@ -67,8 +67,8 @@ func TestExecuteReconcileStateDryRunReportsWithoutWriting(t *testing.T) {
|
||||
if !strings.Contains(stdout.String(), "status=would_change") {
|
||||
t.Fatalf("stdout = %q, want would_change status", stdout.String())
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,summary.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
@@ -173,7 +173,10 @@ func writeReconcileStateLocalFixture(t *testing.T) (string, string, string) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
|
||||
writeCatalogDestinationState(t, destinationRoot, manifest, true)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "html.txt"), []byte("other"), 0o600); err != nil {
|
||||
t.Fatalf("write other owner output: %v", err)
|
||||
}
|
||||
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||
return sourceRoot, destinationRoot, configPath
|
||||
}
|
||||
|
||||
@@ -634,8 +634,8 @@ func TestExecuteRunDryRun(t *testing.T) {
|
||||
}
|
||||
wantStdout := "Configured pipelines: 1\n" +
|
||||
"- pipeline=reports source=local bundles=1 destinations=archive\n" +
|
||||
" - bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt reason=\"destination state is absent\"\n" +
|
||||
"Final status: ok planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true fixed_path=0\n"
|
||||
" - bundle=. destination=archive backend=local action=publish_new workflow=additive outputs=report.md,summary.txt reason=\"\"\n" +
|
||||
"Final status: ok planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true fixed_path=0\n"
|
||||
if got := stdout.String(); got != wantStdout {
|
||||
t.Fatalf("stdout = %q, want %q", got, wantStdout)
|
||||
}
|
||||
@@ -693,6 +693,42 @@ func TestExecuteRunJSONDryRun(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunJSONReportsSkipSame(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||
|
||||
var firstStdout, firstStderr bytes.Buffer
|
||||
if code := Execute(context.Background(), []string{"run", "--config", configPath}, &firstStdout, &firstStderr); code != exitOK {
|
||||
t.Fatalf("first exit code = %d, want %d; stderr = %q", code, exitOK, firstStderr.String())
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := Execute(context.Background(), []string{"run", "--config", configPath, "--format", "json"}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
envelope := decodeEnvelope(t, &stdout)
|
||||
result := envelopeResult(t, envelope)
|
||||
actions, ok := result["actions"].([]any)
|
||||
if !ok || len(actions) != 1 {
|
||||
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||
}
|
||||
action, ok := actions[0].(map[string]any)
|
||||
if !ok || action["action"] != "skip_same" {
|
||||
t.Fatalf("action = %#v, want skip_same", actions[0])
|
||||
}
|
||||
summary, ok := result["summary"].(map[string]any)
|
||||
if !ok || summary["skip_same"] != float64(1) || summary["publish_new"] != float64(0) {
|
||||
t.Fatalf("summary = %#v, want skip_same counter", result["summary"])
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunJSONDryRunReportsFixedPathMapping(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -871,8 +907,8 @@ func TestExecuteRunJSONPartialFailure(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged file: %v", err)
|
||||
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged planned file: %v", err)
|
||||
}
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte(`
|
||||
@@ -938,8 +974,8 @@ func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "action=force_replace") {
|
||||
t.Fatalf("stdout = %q, want force_replace", stdout.String())
|
||||
if !strings.Contains(stdout.String(), "action=force_replace workflow=additive") || !strings.Contains(stdout.String(), "force_replace=1") {
|
||||
t.Fatalf("stdout = %q, want forced replacement", stdout.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); err != nil {
|
||||
t.Fatalf("unmanaged file stat error = %v", err)
|
||||
|
||||
@@ -55,10 +55,8 @@ type Destination struct {
|
||||
Transform Transform `yaml:"transform"`
|
||||
PathMap PathMapping `yaml:"path_mapping"`
|
||||
Links *Links `yaml:"links"`
|
||||
State StatePolicy `yaml:"state"`
|
||||
Reconciliation ReconciliationPolicy `yaml:"reconciliation"`
|
||||
Workflow string `yaml:"workflow"`
|
||||
Retention RetentionPolicy `yaml:"retention"`
|
||||
Transfer TransferPolicy `yaml:"transfer"`
|
||||
}
|
||||
|
||||
type Backend struct {
|
||||
@@ -110,6 +108,7 @@ type MarkdownToHTML struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Mode string `yaml:"mode"`
|
||||
Input string `yaml:"input"`
|
||||
CssHref string `yaml:"css_href"`
|
||||
}
|
||||
|
||||
type PathMapping struct {
|
||||
@@ -121,14 +120,6 @@ type Links struct {
|
||||
Primary string `yaml:"primary"`
|
||||
}
|
||||
|
||||
type ReconciliationPolicy struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type StatePolicy struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type RetentionPolicy struct {
|
||||
Prune PrunePolicy `yaml:"prune"`
|
||||
}
|
||||
@@ -138,10 +129,3 @@ type PrunePolicy struct {
|
||||
OlderThan *Duration `yaml:"older_than"`
|
||||
KeepLatest *int `yaml:"keep_latest"`
|
||||
}
|
||||
|
||||
type TransferPolicy struct {
|
||||
OnDestinationSame string `yaml:"on_destination_same"`
|
||||
OnDestinationOlder string `yaml:"on_destination_older"`
|
||||
OnDestinationNewer string `yaml:"on_destination_newer"`
|
||||
OnConflict string `yaml:"on_conflict"`
|
||||
}
|
||||
|
||||
@@ -20,12 +20,6 @@ const (
|
||||
ValidationActionFail = "fail"
|
||||
)
|
||||
|
||||
const (
|
||||
TransferActionSkip = "skip"
|
||||
TransferActionReplace = "replace"
|
||||
TransferActionFail = "fail"
|
||||
)
|
||||
|
||||
const (
|
||||
TransformModeSidecar = transform.MarkdownModeSidecar
|
||||
TransformModeIndex = transform.MarkdownModeIndex
|
||||
@@ -43,13 +37,8 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
ReconciliationModeReplace = "replace"
|
||||
ReconciliationModeMerge = "merge"
|
||||
)
|
||||
|
||||
const (
|
||||
StateModeSingleOwner = "single_owner"
|
||||
StateModeSharedRoot = "shared_root"
|
||||
WorkflowAdditive = "additive"
|
||||
WorkflowReplacement = "replacement"
|
||||
)
|
||||
|
||||
const DefaultS3Region = "us-east-1"
|
||||
@@ -89,23 +78,8 @@ func ApplyDefaults(cfg *Config) {
|
||||
if destination.Links != nil && destination.Links.Primary == "" {
|
||||
destination.Links.Primary = LinkPrimaryAuto
|
||||
}
|
||||
if destination.State.Mode == "" {
|
||||
destination.State.Mode = StateModeSingleOwner
|
||||
}
|
||||
if destination.Reconciliation.Mode == "" {
|
||||
destination.Reconciliation.Mode = ReconciliationModeReplace
|
||||
}
|
||||
if destination.Transfer.OnDestinationSame == "" {
|
||||
destination.Transfer.OnDestinationSame = TransferActionSkip
|
||||
}
|
||||
if destination.Transfer.OnDestinationOlder == "" {
|
||||
destination.Transfer.OnDestinationOlder = TransferActionReplace
|
||||
}
|
||||
if destination.Transfer.OnDestinationNewer == "" {
|
||||
destination.Transfer.OnDestinationNewer = TransferActionSkip
|
||||
}
|
||||
if destination.Transfer.OnConflict == "" {
|
||||
destination.Transfer.OnConflict = TransferActionFail
|
||||
if destination.Workflow == "" {
|
||||
destination.Workflow = WorkflowAdditive
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,14 +30,8 @@ pipelines:
|
||||
if got, want := cfg.Pipelines[0].Validation.OnDigestMismatch, ValidationActionFail; got != want {
|
||||
t.Fatalf("validation default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
|
||||
t.Fatalf("transfer default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.Reconciliation.Mode, ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode default = %q, want %q", got, want)
|
||||
if got, want := destination.Workflow, WorkflowAdditive; got != want {
|
||||
t.Fatalf("workflow default = %q, want %q", got, want)
|
||||
}
|
||||
if destination.Retention.Prune.Enabled {
|
||||
t.Fatal("retention.prune.enabled default = true, want false")
|
||||
@@ -173,7 +167,7 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsExplicitReconciliationModes(t *testing.T) {
|
||||
func TestLoadFileAcceptsExplicitWorkflows(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
@@ -184,50 +178,19 @@ pipelines:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
reconciliation:
|
||||
mode: replace
|
||||
workflow: additive
|
||||
- id: web
|
||||
backend: local
|
||||
path: /web
|
||||
reconciliation:
|
||||
mode: merge
|
||||
workflow: replacement
|
||||
`)
|
||||
|
||||
destinations := cfg.Pipelines[0].Destinations
|
||||
if got, want := destinations[0].Reconciliation.Mode, ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("archive reconciliation mode = %q, want %q", got, want)
|
||||
if got, want := destinations[0].Workflow, WorkflowAdditive; got != want {
|
||||
t.Fatalf("archive workflow = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destinations[1].Reconciliation.Mode, ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("web reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsExplicitStateModes(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
state:
|
||||
mode: single_owner
|
||||
- id: web
|
||||
backend: local
|
||||
path: /web
|
||||
state:
|
||||
mode: shared_root
|
||||
`)
|
||||
|
||||
destinations := cfg.Pipelines[0].Destinations
|
||||
if got, want := destinations[0].State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("archive state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destinations[1].State.Mode, StateModeSharedRoot; got != want {
|
||||
t.Fatalf("web state mode = %q, want %q", got, want)
|
||||
if got, want := destinations[1].Workflow, WorkflowReplacement; got != want {
|
||||
t.Fatalf("web workflow = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -852,8 +815,63 @@ pipelines:
|
||||
`, "backend ftp is unsupported")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidTransferAction(t *testing.T) {
|
||||
func TestLoadFileRejectsInvalidWorkflow(t *testing.T) {
|
||||
assertLoadError(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
workflow: append
|
||||
`, "workflow must be additive or replacement")
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsLegacyDestinationPolicyFields(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"state": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
state:
|
||||
mode: single_owner
|
||||
`,
|
||||
"reconciliation": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
reconciliation:
|
||||
mode: replace
|
||||
`,
|
||||
"takeover": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
takeover:
|
||||
mode: same_pipeline
|
||||
`,
|
||||
"transfer": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
@@ -864,8 +882,14 @@ pipelines:
|
||||
backend: local
|
||||
path: /archive
|
||||
transfer:
|
||||
on_destination_older: overwrite
|
||||
`, "on_destination_older must be replace or fail")
|
||||
on_destination_older: replace
|
||||
`,
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assertLoadError(t, body, "field "+name+" not found")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidValidationAction(t *testing.T) {
|
||||
@@ -934,19 +958,14 @@ pipelines:
|
||||
}
|
||||
|
||||
func TestExampleConfigsLoad(t *testing.T) {
|
||||
for _, path := range []string{
|
||||
"../../examples/local-to-local.yml",
|
||||
"../../examples/local-publish.yml",
|
||||
"../../examples/local-html.yml",
|
||||
"../../examples/local-index.yml",
|
||||
"../../examples/fan-out.yml",
|
||||
"../../examples/archive-and-latest.yml",
|
||||
"../../examples/merge-reconciliation.yml",
|
||||
"../../examples/shared-root.yml",
|
||||
"../../examples/http-upload-local.yml",
|
||||
"../../examples/ssh-destination.yml",
|
||||
"../../examples/s3-destination.yml",
|
||||
} {
|
||||
paths, err := filepath.Glob("../../examples/*.yml")
|
||||
if err != nil {
|
||||
t.Fatalf("glob examples: %v", err)
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
t.Fatal("no example configs found")
|
||||
}
|
||||
for _, path := range paths {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
if _, err := LoadFile(path); err != nil {
|
||||
t.Fatalf("LoadFile(%q) error = %v", path, err)
|
||||
|
||||
@@ -2,8 +2,10 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
)
|
||||
@@ -72,10 +74,8 @@ func Validate(cfg Config) error {
|
||||
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
|
||||
errs = validatePathMapping(errs, destinationContext+".path_mapping", destination.PathMap)
|
||||
errs = validateLinks(errs, destinationContext+".links", destination.Links)
|
||||
errs = validateStatePolicy(errs, destinationContext+".state", destination.State)
|
||||
errs = validateReconciliationPolicy(errs, destinationContext+".reconciliation", destination.Reconciliation)
|
||||
errs = validateWorkflow(errs, destinationContext+".workflow", destination.Workflow)
|
||||
errs = validateRetentionPolicy(errs, destinationContext+".retention", destination.Retention)
|
||||
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -281,9 +281,15 @@ func ValidatePublishTransformPolicy(publish PublishPolicy, transform Transform)
|
||||
if transform.MarkdownToHTML.Input != "" && !transform.MarkdownToHTML.Enabled {
|
||||
return fmt.Errorf("transform.markdown_to_html.input requires transform.markdown_to_html.enabled to be true")
|
||||
}
|
||||
if transform.MarkdownToHTML.CssHref != "" && !transform.MarkdownToHTML.Enabled {
|
||||
return fmt.Errorf("transform.markdown_to_html.css_href requires transform.markdown_to_html.enabled to be true")
|
||||
}
|
||||
if transform.MarkdownToHTML.Input != "" && mode != TransformModeIndex {
|
||||
return fmt.Errorf("transform.markdown_to_html.input is only valid when mode is %s", TransformModeIndex)
|
||||
}
|
||||
if err := validateCSSHref(transform.MarkdownToHTML.CssHref); err != nil {
|
||||
return fmt.Errorf("transform.markdown_to_html.css_href %w", err)
|
||||
}
|
||||
if transform.MarkdownToHTML.Enabled && !publish.HTML {
|
||||
return fmt.Errorf("transform.markdown_to_html.enabled requires publish.html to be true")
|
||||
}
|
||||
@@ -293,6 +299,49 @@ func ValidatePublishTransformPolicy(publish PublishPolicy, transform Transform)
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCSSHref(value string) error {
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
for _, character := range value {
|
||||
if unicode.IsControl(character) || unicode.IsSpace(character) {
|
||||
return fmt.Errorf("must not contain whitespace or control characters")
|
||||
}
|
||||
}
|
||||
if strings.ContainsAny(value, "\\<>\"'") {
|
||||
return fmt.Errorf("must not contain backslashes or HTML-sensitive characters")
|
||||
}
|
||||
if strings.HasPrefix(value, "//") {
|
||||
return fmt.Errorf("must not be scheme-relative")
|
||||
}
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("must be a valid URL reference: %w", err)
|
||||
}
|
||||
if parsed.Fragment != "" {
|
||||
return fmt.Errorf("must not include a fragment")
|
||||
}
|
||||
if parsed.Scheme != "" {
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("scheme must be http or https")
|
||||
}
|
||||
if parsed.Host == "" {
|
||||
return fmt.Errorf("host is required for absolute URLs")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return fmt.Errorf("must not include userinfo")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if parsed.Host != "" {
|
||||
return fmt.Errorf("must not be scheme-relative")
|
||||
}
|
||||
if parsed.Path == "" {
|
||||
return fmt.Errorf("relative URL path is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePathMapping(errs ValidationErrors, context string, mapping PathMapping) ValidationErrors {
|
||||
if mapping.Mode != PathMappingPreserveRelative && mapping.Mode != PathMappingFixed {
|
||||
errs = append(errs, context+".mode must be "+PathMappingPreserveRelative+" or "+PathMappingFixed)
|
||||
@@ -317,9 +366,9 @@ func validateLinks(errs ValidationErrors, context string, links *Links) Validati
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateReconciliationPolicy(errs ValidationErrors, context string, policy ReconciliationPolicy) ValidationErrors {
|
||||
if policy.Mode != ReconciliationModeReplace && policy.Mode != ReconciliationModeMerge {
|
||||
errs = append(errs, context+".mode must be "+ReconciliationModeReplace+" or "+ReconciliationModeMerge)
|
||||
func validateWorkflow(errs ValidationErrors, context, workflow string) ValidationErrors {
|
||||
if workflow != WorkflowAdditive && workflow != WorkflowReplacement {
|
||||
errs = append(errs, context+" must be "+WorkflowAdditive+" or "+WorkflowReplacement)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
@@ -340,26 +389,3 @@ func validateRetentionPolicy(errs ValidationErrors, context string, policy Reten
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateStatePolicy(errs ValidationErrors, context string, policy StatePolicy) ValidationErrors {
|
||||
if policy.Mode != StateModeSingleOwner && policy.Mode != StateModeSharedRoot {
|
||||
errs = append(errs, context+".mode must be "+StateModeSingleOwner+" or "+StateModeSharedRoot)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
|
||||
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
|
||||
errs = append(errs, context+".on_destination_same must be skip or fail")
|
||||
}
|
||||
if policy.OnDestinationOlder != TransferActionReplace && policy.OnDestinationOlder != TransferActionFail {
|
||||
errs = append(errs, context+".on_destination_older must be replace or fail")
|
||||
}
|
||||
if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail && policy.OnDestinationNewer != TransferActionReplace {
|
||||
errs = append(errs, context+".on_destination_newer must be skip, replace, or fail")
|
||||
}
|
||||
if policy.OnConflict != TransferActionFail && policy.OnConflict != TransferActionReplace {
|
||||
errs = append(errs, context+".on_conflict must be fail or replace")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
@@ -51,29 +51,6 @@ func TestValidateChecksPublishTransformPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAcceptsForceReplacementTransferActions(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{
|
||||
Backend: BackendLocal,
|
||||
Path: "/source",
|
||||
},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Transfer: TransferPolicy{
|
||||
OnDestinationNewer: TransferActionReplace,
|
||||
OnConflict: TransferActionReplace,
|
||||
},
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePathMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -108,15 +85,15 @@ func TestValidatePathMapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReconciliationPolicy(t *testing.T) {
|
||||
func TestValidateWorkflow(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
workflow string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "replace", mode: ReconciliationModeReplace},
|
||||
{name: "merge", mode: ReconciliationModeMerge},
|
||||
{name: "invalid", mode: "append", wantErr: true},
|
||||
{name: "additive", workflow: WorkflowAdditive},
|
||||
{name: "replacement", workflow: WorkflowReplacement},
|
||||
{name: "invalid", workflow: "append", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -127,7 +104,7 @@ func TestValidateReconciliationPolicy(t *testing.T) {
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Reconciliation: ReconciliationPolicy{Mode: tt.mode},
|
||||
Workflow: tt.workflow,
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
@@ -142,18 +119,7 @@ func TestValidateReconciliationPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateStatePolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "single owner", mode: StateModeSingleOwner},
|
||||
{name: "shared root", mode: StateModeSharedRoot},
|
||||
{name: "invalid", mode: "shared", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
func TestValidateWorkflowReportsFieldContext(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||
@@ -161,18 +127,17 @@ func TestValidateStatePolicy(t *testing.T) {
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
State: StatePolicy{Mode: tt.mode},
|
||||
Workflow: "append",
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
err := Validate(cfg)
|
||||
if tt.wantErr && err == nil {
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
want := "pipelines[0].destinations[0].workflow must be additive or replacement"
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("Validate() error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -328,6 +293,24 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
|
||||
Input: "report.md",
|
||||
}},
|
||||
},
|
||||
{
|
||||
name: "html only sidecar css href allowed",
|
||||
publish: PublishPolicy{HTML: true},
|
||||
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||
Enabled: true,
|
||||
Mode: TransformModeSidecar,
|
||||
CssHref: "/assets/report.css",
|
||||
}},
|
||||
},
|
||||
{
|
||||
name: "html only index css href allowed",
|
||||
publish: PublishPolicy{HTML: true},
|
||||
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||
Enabled: true,
|
||||
Mode: TransformModeIndex,
|
||||
CssHref: "assets/report.css?v=20260614",
|
||||
}},
|
||||
},
|
||||
{
|
||||
name: "source and html sidecar allowed",
|
||||
publish: PublishPolicy{Source: true, HTML: true},
|
||||
@@ -425,6 +408,16 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
|
||||
}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "disabled markdown css href rejected",
|
||||
publish: PublishPolicy{Source: true},
|
||||
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
|
||||
Enabled: false,
|
||||
Mode: TransformModeSidecar,
|
||||
CssHref: "/assets/report.css",
|
||||
}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "disabled markdown wrong mode rejected",
|
||||
publish: PublishPolicy{Source: true},
|
||||
@@ -436,3 +429,44 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCSSHref(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
value string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "empty"},
|
||||
{name: "root relative", value: "/assets/report.css"},
|
||||
{name: "relative", value: "assets/report.css"},
|
||||
{name: "parent relative", value: "../assets/report.css"},
|
||||
{name: "query", value: "/assets/report.css?v=20260614"},
|
||||
{name: "http", value: "http://example.com/report.css"},
|
||||
{name: "https", value: "https://example.com/assets/report.css?v=1"},
|
||||
{name: "javascript", value: "javascript:alert(1)", wantErr: true},
|
||||
{name: "data", value: "data:text/css,body{}", wantErr: true},
|
||||
{name: "file", value: "file:///tmp/report.css", wantErr: true},
|
||||
{name: "scheme relative", value: "//example.com/report.css", wantErr: true},
|
||||
{name: "userinfo", value: "https://user@example.com/report.css", wantErr: true},
|
||||
{name: "fragment", value: "/assets/report.css#main", wantErr: true},
|
||||
{name: "space", value: "/assets/report css", wantErr: true},
|
||||
{name: "tab", value: "/assets/report\tcss", wantErr: true},
|
||||
{name: "newline", value: "/assets/report\ncss", wantErr: true},
|
||||
{name: "backslash", value: `assets\report.css`, wantErr: true},
|
||||
{name: "less than", value: "/assets/<report>.css", wantErr: true},
|
||||
{name: "double quote", value: `/assets/"report".css`, wantErr: true},
|
||||
{name: "single quote", value: "/assets/'report'.css", wantErr: true},
|
||||
{name: "query only", value: "?v=1", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateCSSHref(tt.value)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("validateCSSHref() error = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("validateCSSHref() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,142 +4,24 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
||||
switch plan.Action {
|
||||
case ActionSkipSame, ActionSkipDestinationNewer:
|
||||
case ActionSkipSame:
|
||||
return nil
|
||||
case ActionPublishNew, ActionReplaceOlder, ActionForceReplace:
|
||||
if usesSharedRootState(req, plan) {
|
||||
return executeSharedRoot(ctx, req, plan)
|
||||
}
|
||||
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog, ActionForceReplace:
|
||||
return executeCatalog(ctx, req, plan)
|
||||
default:
|
||||
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
||||
}
|
||||
|
||||
if plan.Action == ActionReplaceOlder {
|
||||
if plan.ExistingState == nil {
|
||||
return fmt.Errorf("replace requires existing destination state")
|
||||
}
|
||||
if plan.Reconciliation.Mode == config.ReconciliationModeReplace {
|
||||
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if plan.Action == ActionForceReplace {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if usesMergeRetention(plan) {
|
||||
if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
cleanup := func() {
|
||||
outputs := writtenOutputs
|
||||
if usesMergeRetention(plan) {
|
||||
outputs = newOutputs
|
||||
}
|
||||
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||
}
|
||||
for _, output := range plan.Outputs {
|
||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data := output.Data
|
||||
if output.Kind == state.OutputKindSource {
|
||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
}
|
||||
managed := outputManagedByExistingState(output, plan.ExistingState)
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
writtenOutputs = append(writtenOutputs, output)
|
||||
if !managed {
|
||||
newOutputs = append(newOutputs, output)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
createdAt := now
|
||||
if plan.ExistingState != nil {
|
||||
createdAt = plan.ExistingState.CreatedAt
|
||||
}
|
||||
stateOutputs, err := stateOutputsForPlan(plan, now)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
destinationState := state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
DistributorVersion: req.DistributorVersion,
|
||||
PipelineID: req.PipelineID,
|
||||
DestinationID: req.DestinationID,
|
||||
PublishedAt: now,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
||||
Outputs: stateOutputs,
|
||||
}
|
||||
if plan.PrimaryURL != "" {
|
||||
destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
||||
}
|
||||
if err := state.Validate(destinationState); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
||||
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
|
||||
if plan.Action == ActionForceReplace {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
@@ -147,18 +29,26 @@ func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if plan.Action == ActionReplaceOlder && plan.Reconciliation.Mode == config.ReconciliationModeReplace {
|
||||
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
} else if plan.Action == ActionReplaceCatalog {
|
||||
if plan.ClearDestinationRoot {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if len(plan.CatalogOutputsToDelete) > 0 {
|
||||
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, catalogOutputPaths(plan.CatalogOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
cleanup := func() {
|
||||
outputs := writtenOutputs
|
||||
if plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
if plan.Action == ActionUpsertAdditive {
|
||||
outputs = newOutputs
|
||||
}
|
||||
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||
@@ -169,6 +59,11 @@ func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
created, err := catalogWriteCreatesOutput(ctx, req.DestinationBackend, destinationPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data := output.Data
|
||||
if output.Kind == state.OutputKindSource {
|
||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
||||
@@ -182,28 +77,22 @@ func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
managed := outputManagedBySharedRootPlan(output, plan)
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil {
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: catalogOutputOverwriteAllowed(plan, output), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
writtenOutputs = append(writtenOutputs, output)
|
||||
if !managed {
|
||||
if created {
|
||||
newOutputs = append(newOutputs, output)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
sharedRootState, err := sharedRootStateForPlan(req, plan, now)
|
||||
if err != nil {
|
||||
catalogState := catalogStateForPlan(req, plan)
|
||||
if err := state.ValidateCatalog(catalogState); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if err := state.ValidateSharedRoot(sharedRootState); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(sharedRootState, "", " ")
|
||||
data, err := json.MarshalIndent(catalogState, "", " ")
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
@@ -214,165 +103,73 @@ func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: catalogStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error {
|
||||
for _, output := range plan.Outputs {
|
||||
if outputManagedByExistingState(output, plan.ExistingState) {
|
||||
continue
|
||||
}
|
||||
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
func catalogWriteCreatesOutput(ctx context.Context, backend storage.Backend, destinationPath string) (bool, error) {
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath))
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
return false, nil
|
||||
} else if storage.IsNotFound(err) {
|
||||
return true, nil
|
||||
} else {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func usesSharedRootState(req Request, plan Plan) bool {
|
||||
return normalizeState(req.State).Mode == config.StateModeSharedRoot || plan.StateMode == config.StateModeSharedRoot
|
||||
}
|
||||
|
||||
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
|
||||
if existing == nil {
|
||||
func catalogOutputOverwriteAllowed(plan Plan, output Output) bool {
|
||||
if plan.ClearDestinationRoot {
|
||||
return false
|
||||
}
|
||||
_, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath)
|
||||
if plan.SupersededLegacy != nil {
|
||||
return true
|
||||
}
|
||||
if plan.ExistingCatalog == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := state.FindCatalogOutputByPath(plan.ExistingCatalog.Outputs, output.DestinationPath)
|
||||
return ok
|
||||
}
|
||||
|
||||
func outputManagedBySharedRootPlan(output Output, plan Plan) bool {
|
||||
for _, existing := range currentOwnerSharedRootOutputs(plan) {
|
||||
if existing.Path == output.DestinationPath {
|
||||
return true
|
||||
func catalogStateForPlan(req Request, plan Plan) state.CatalogState {
|
||||
now := requestTime(req)
|
||||
createdAt := now
|
||||
if plan.ExistingCatalog != nil {
|
||||
createdAt = plan.ExistingCatalog.CreatedAt
|
||||
}
|
||||
outputs := make([]state.CatalogOutputFile, 0, len(plan.CatalogOutputsToRetain)+len(plan.CatalogOutputsToWrite))
|
||||
outputs = append(outputs, plan.CatalogOutputsToRetain...)
|
||||
outputs = append(outputs, plan.CatalogOutputsToWrite...)
|
||||
sort.SliceStable(outputs, func(i, j int) bool {
|
||||
if outputs[i].Path != outputs[j].Path {
|
||||
return outputs[i].Path < outputs[j].Path
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) {
|
||||
existingOutputs := []state.OutputFile(nil)
|
||||
if plan.ExistingState != nil {
|
||||
existingOutputs = plan.ExistingState.Outputs
|
||||
if outputs[i].PipelineID != outputs[j].PipelineID {
|
||||
return outputs[i].PipelineID < outputs[j].PipelineID
|
||||
}
|
||||
planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now)
|
||||
if !usesMergeRetention(plan) || plan.ExistingState == nil {
|
||||
return planned, nil
|
||||
}
|
||||
return state.MergeOutputFiles(plan.ExistingState.Outputs, planned)
|
||||
}
|
||||
|
||||
func usesMergeRetention(plan Plan) bool {
|
||||
return plan.Reconciliation.Mode == config.ReconciliationModeMerge && plan.Action == ActionReplaceOlder
|
||||
}
|
||||
|
||||
func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) {
|
||||
now = now.UTC()
|
||||
scope := plan.OwnerScope
|
||||
if scope.PipelineID == "" && scope.DestinationID == "" {
|
||||
scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
}
|
||||
base := sharedRootBaseState(req, plan, now)
|
||||
owner := state.OwnerRecord{
|
||||
Scope: scope,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
||||
}
|
||||
if plan.PrimaryURL != "" {
|
||||
owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
||||
}
|
||||
planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now)
|
||||
if plan.Action == ActionReplaceOlder && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
return state.MergeOwnerOutputs(base, scope, owner, planned)
|
||||
}
|
||||
return state.ReplaceOwnerOutputs(base, scope, owner, planned)
|
||||
}
|
||||
|
||||
func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState {
|
||||
if plan.Action == ActionForceReplace {
|
||||
return newSharedRootState(req, now)
|
||||
}
|
||||
if plan.ExistingSharedRoot != nil {
|
||||
base := *plan.ExistingSharedRoot
|
||||
base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...)
|
||||
base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...)
|
||||
base.DistributorVersion = req.DistributorVersion
|
||||
base.UpdatedAt = now
|
||||
return base
|
||||
}
|
||||
if plan.ExistingState != nil {
|
||||
base := newSharedRootState(req, now)
|
||||
base.CreatedAt = plan.ExistingState.CreatedAt
|
||||
base.UpdatedAt = now
|
||||
return base
|
||||
}
|
||||
return newSharedRootState(req, now)
|
||||
}
|
||||
|
||||
func newSharedRootState(req Request, now time.Time) state.SharedRootState {
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: req.DistributorVersion,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{},
|
||||
Outputs: []state.SharedRootOutputFile{},
|
||||
}
|
||||
}
|
||||
|
||||
func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile {
|
||||
if plan.ExistingSharedRoot != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs))
|
||||
for _, output := range plan.ExistingSharedRoot.Outputs {
|
||||
if output.Owner == plan.OwnerScope {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
if plan.ExistingState != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs))
|
||||
for _, output := range plan.ExistingState.Outputs {
|
||||
outputs = append(outputs, state.SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: plan.OwnerScope,
|
||||
SourceID: plan.ExistingState.Source.Manifest.ID,
|
||||
SourceDigest: plan.ExistingState.Source.Manifest.Digest,
|
||||
SourceCreated: plan.ExistingState.Source.Manifest.Created,
|
||||
CreatedAt: output.CreatedAt,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
return outputs[i].DestinationID < outputs[j].DestinationID
|
||||
})
|
||||
return state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: req.DistributorVersion,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: outputs,
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string {
|
||||
func catalogStateWriteOverwrites(plan Plan) bool {
|
||||
return plan.ExistingCatalog != nil || plan.SupersededLegacy != nil
|
||||
}
|
||||
|
||||
func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func sharedRootStateWriteOverwrites(plan Plan) bool {
|
||||
return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace
|
||||
}
|
||||
|
||||
@@ -2,99 +2,26 @@ package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
)
|
||||
|
||||
func TestExecuteCleansUpAfterWriteFailure(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "summary.txt"}
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{})
|
||||
req := Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: "",
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Transfer: config.TransferPolicy{OnDestinationSame: config.TransferActionSkip, OnDestinationOlder: config.TransferActionReplace, OnDestinationNewer: config.TransferActionSkip, OnConflict: config.TransferActionFail},
|
||||
DistributorVersion: "test",
|
||||
func TestExecuteAdditiveWritesOutputsAndCatalog(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want error")
|
||||
}
|
||||
found, err := destinationBackend.HasAny(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("HasAny() error = %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Fatal("destination has content after failed execution")
|
||||
}
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "old.txt", "retained")
|
||||
|
||||
func TestExecuteReplaceDeletesOmittedManagedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteMergeRetainsOmittedAndOverwritesManagedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
existingState := testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
@@ -102,256 +29,199 @@ func TestExecuteMergeRetainsOmittedAndOverwritesManagedOutputs(t *testing.T) {
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
if got, want := len(outputs), 2; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "old.txt", "retained")
|
||||
catalog := readCatalogState(t, destinationBackend, "")
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
|
||||
}
|
||||
if !outputs["summary.txt"].CreatedAt.Equal(existingState.Outputs[1].CreatedAt) || !outputs["summary.txt"].UpdatedAt.Equal(existingState.Outputs[1].UpdatedAt) {
|
||||
t.Fatalf("retained output timestamps = %#v, want existing %#v", outputs["summary.txt"], existingState.Outputs[1])
|
||||
if len(catalog.Outputs) != 3 {
|
||||
t.Fatalf("catalog outputs = %#v, want three outputs", catalog.Outputs)
|
||||
}
|
||||
if !outputs["report.md"].CreatedAt.Equal(existingState.Outputs[0].CreatedAt) || !outputs["report.md"].UpdatedAt.After(existingState.Outputs[0].UpdatedAt) {
|
||||
t.Fatalf("updated output timestamps = %#v, want preserved created_at and newer updated_at", outputs["report.md"])
|
||||
report, ok := state.FindCatalogOutputByPath(catalog.Outputs, "report.md")
|
||||
if !ok {
|
||||
t.Fatalf("catalog outputs = %#v, want report.md", catalog.Outputs)
|
||||
}
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
if !report.CreatedAt.Equal(planCreatedAt) || !report.UpdatedAt.Equal(planUpdatedAt) {
|
||||
t.Fatalf("report times = %s/%s, want created preserved and updated now", report.CreatedAt, report.UpdatedAt)
|
||||
}
|
||||
if report.SourcePath != "" {
|
||||
t.Fatalf("source catalog output source_path = %q, want empty", report.SourcePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteMergeFailsOnUnmanagedDestinationPathCollision(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = []bundle.ManifestFile{{
|
||||
Path: "summary.txt",
|
||||
SHA256: bundle.FileDigest([]byte("Summary\n")),
|
||||
Size: int64(len("Summary\n")),
|
||||
}}
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
func TestExecuteReplacementDeletesCurrentOwnerAndPreservesOtherOwners(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "stale.txt", "delete me")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "shared.txt", "keep me")
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want unmanaged path collision")
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "stale.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "shared.txt", "keep me")
|
||||
catalog := readCatalogState(t, destinationBackend, "")
|
||||
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "stale.txt"); ok {
|
||||
t.Fatalf("catalog outputs = %#v, want stale.txt removed", catalog.Outputs)
|
||||
}
|
||||
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "shared.txt"); !ok {
|
||||
t.Fatalf("catalog outputs = %#v, want shared.txt retained", catalog.Outputs)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
|
||||
}
|
||||
|
||||
func TestExecuteMergeFailureCleansUpOnlyNewOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{
|
||||
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||
{Path: "new.md", Data: "new\n"},
|
||||
{Path: "fail.md", Data: "fail\n"},
|
||||
},
|
||||
})
|
||||
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "fail.md"}
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = []bundle.ManifestFile{sourceBundle.Manifest.Files[0]}
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend.Backend, "", older, testutil.DestinationStateOptions{})
|
||||
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
req.DestinationBundlePath = "bundle"
|
||||
legacyState := legacyStateDocument(2)
|
||||
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want injected failure")
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend.Backend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend.Backend, "new.md")
|
||||
testutil.AssertFakeMissing(t, destinationBackend.Backend, "fail.md")
|
||||
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unplanned.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||
readCatalogState(t, destinationBackend, "bundle")
|
||||
}
|
||||
|
||||
func TestExecuteFixedPathSupportsReconciliationModes(t *testing.T) {
|
||||
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "new", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.ID = "older.source"
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
legacyState := legacyStateDocument(2)
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
||||
catalog := readCatalogState(t, destinationBackend, "")
|
||||
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "unplanned.txt"); ok {
|
||||
t.Fatalf("catalog outputs = %#v, want unplanned file omitted", catalog.Outputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteForceReplaceClearsDestinationBundlePathOnly(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
req.DestinationBundlePath = "bundle"
|
||||
req.Force = true
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/report.md", "old report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle-child/keep.txt", "keep")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionForceReplace)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/summary.txt", "Summary\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unplanned.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle-child/keep.txt", "keep")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||
catalog := readCatalogState(t, destinationBackend, "bundle")
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
|
||||
}
|
||||
if len(catalog.Outputs) != 2 {
|
||||
t.Fatalf("catalog outputs = %#v, want planned outputs only", catalog.Outputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteForceReplaceClearsFixedDestinationRoot(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
req.DestinationBundlePath = ""
|
||||
req.PathMapping = config.PathMappingFixed
|
||||
req.Force = true
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "remove")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
if plan.Action != ActionForceReplace || storage.DisplayPath(plan.DestinationBundlePath) != "." {
|
||||
t.Fatalf("plan action=%s destination=%s, want force_replace at root", plan.Action, storage.DisplayPath(plan.DestinationBundlePath))
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
if got, want := destinationState.Reconciliation.Mode, mode; got != want {
|
||||
t.Fatalf("state reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "unplanned.txt")
|
||||
readCatalogState(t, destinationBackend, "")
|
||||
}
|
||||
|
||||
func TestExecuteReconciliationModesHonorPublishPolicies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
publish config.PublishPolicy
|
||||
transform config.Transform
|
||||
transformer TransformerResolver
|
||||
wantPaths []string
|
||||
}{
|
||||
{
|
||||
name: "source only",
|
||||
publish: config.PublishPolicy{Source: true},
|
||||
wantPaths: []string{"report.md"},
|
||||
},
|
||||
{
|
||||
name: "generated only",
|
||||
publish: config.PublishPolicy{HTML: true},
|
||||
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
|
||||
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
|
||||
Path: "report.html",
|
||||
SourcePath: "report.md",
|
||||
Transform: transform.MarkdownToHTML,
|
||||
Data: []byte("<h1>Report</h1>\n"),
|
||||
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
|
||||
Size: int64(len("<h1>Report</h1>\n")),
|
||||
}}}},
|
||||
wantPaths: []string{"report.html"},
|
||||
},
|
||||
{
|
||||
name: "source and generated",
|
||||
publish: config.PublishPolicy{Source: true, HTML: true},
|
||||
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
|
||||
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
|
||||
Path: "report.html",
|
||||
SourcePath: "report.md",
|
||||
Transform: transform.MarkdownToHTML,
|
||||
Data: []byte("<h1>Report</h1>\n"),
|
||||
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
|
||||
Size: int64(len("<h1>Report</h1>\n")),
|
||||
}}}},
|
||||
wantPaths: []string{"report.md", "report.html"},
|
||||
},
|
||||
}
|
||||
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
|
||||
for _, tt := range tests {
|
||||
t.Run(mode+" "+tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
|
||||
req.Publish = tt.publish
|
||||
req.Transform = tt.transform
|
||||
req.Transformers = tt.transformer
|
||||
func TestExecuteFailedWriteDoesNotWriteCatalogState(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
if err := destinationBackend.AddDirectory("report.md"); err != nil {
|
||||
t.Fatalf("add conflicting directory: %v", err)
|
||||
}
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
for _, path := range tt.wantPaths {
|
||||
if _, ok := outputs[path]; !ok {
|
||||
t.Fatalf("state outputs = %#v, missing %s", destinationState.Outputs, path)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want write failure")
|
||||
}
|
||||
if _, statErr := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(statErr) {
|
||||
t.Fatalf("state stat error = %v, want missing state", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
type failingBackend struct {
|
||||
*fake.Backend
|
||||
failPath string
|
||||
}
|
||||
|
||||
func (b *failingBackend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
|
||||
if path == b.failPath {
|
||||
return storage.Entry{}, fmt.Errorf("injected write failure")
|
||||
}
|
||||
return b.Backend.WriteFile(ctx, path, data, opts)
|
||||
}
|
||||
|
||||
func (b *failingBackend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
|
||||
if path == b.failPath {
|
||||
return storage.Entry{}, fmt.Errorf("injected write failure")
|
||||
}
|
||||
return b.Backend.WriteFrom(ctx, path, r, opts)
|
||||
}
|
||||
|
||||
func testRequest(sourceBackend storage.Backend, destinationBackend storage.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
|
||||
Transfer: config.TransferPolicy{
|
||||
OnDestinationSame: config.TransferActionSkip,
|
||||
OnDestinationOlder: config.TransferActionReplace,
|
||||
OnDestinationNewer: config.TransferActionSkip,
|
||||
OnConflict: config.TransferActionFail,
|
||||
},
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeState(t *testing.T, backend storage.Backend, bundlePath string) state.DistributorState {
|
||||
func readCatalogState(t *testing.T, backend *fake.Backend, relative string) state.CatalogState {
|
||||
t.Helper()
|
||||
statePath, err := storage.StatePath(bundlePath)
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
data, err := backend.ReadFile(context.Background(), statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse state: %v", err)
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
|
||||
byPath := make(map[string]state.OutputFile, len(outputs))
|
||||
for _, output := range outputs {
|
||||
byPath[output.Path] = output
|
||||
}
|
||||
return byPath
|
||||
return catalog
|
||||
}
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prepare func(t *testing.T, backend *fake.Backend, source bundle.Manifest)
|
||||
transfer config.TransferPolicy
|
||||
wantReason string
|
||||
forceAction bool
|
||||
}{
|
||||
{
|
||||
name: "unmanaged content",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
testutil.WriteFakeFile(t, backend, "bundle/old.txt", "old")
|
||||
},
|
||||
transfer: defaultTransfer(),
|
||||
wantReason: "fail_unmanaged",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "different source id",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
conflict := source
|
||||
conflict.ID = "other.source"
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
},
|
||||
transfer: conflictReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "same created digest conflict",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
conflict := testutil.ValidManifest(testutil.BundleOptions{Files: []testutil.SourceFile{{Path: "report.md", Data: "# Different\n"}}})
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
},
|
||||
transfer: conflictReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "pipeline mismatch",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{PipelineID: "other-pipeline"})
|
||||
},
|
||||
transfer: conflictReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "destination mismatch",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{DestinationID: "other-destination"})
|
||||
},
|
||||
transfer: conflictReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
{
|
||||
name: "newer destination",
|
||||
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
|
||||
t.Helper()
|
||||
newer := source
|
||||
newer.Created = newer.Created.AddDate(0, 0, 1)
|
||||
testutil.WriteFakeDestinationState(t, backend, "bundle", newer, testutil.DestinationStateOptions{})
|
||||
},
|
||||
transfer: newerReplaceTransfer(),
|
||||
wantReason: "requires --force",
|
||||
forceAction: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
tt.prepare(t, destinationBackend, sourceBundle.Manifest)
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, tt.transfer)
|
||||
_, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantReason) {
|
||||
t.Fatalf("Build() error = %v, want %q", err, tt.wantReason)
|
||||
}
|
||||
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if tt.forceAction {
|
||||
if err != nil {
|
||||
t.Fatalf("Build() with force error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace || !plan.Force {
|
||||
t.Fatalf("forced plan action = %s force=%t", plan.Action, plan.Force)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequiresConflictPolicyForStateConflicts(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
conflict := sourceBundle.Manifest
|
||||
conflict.ID = "other.source"
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", conflict, testutil.DestinationStateOptions{})
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
||||
req.Force = true
|
||||
_, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "destination source id differs") {
|
||||
t.Fatalf("Build() error = %v, want conservative conflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteForcedReplacementDeletesOnlyBundlePath(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
|
||||
destinationBackend := fake.New()
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/old.txt", "old")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "outside")
|
||||
|
||||
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace {
|
||||
t.Fatalf("plan action = %s, want force_replace", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.txt")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "outside")
|
||||
}
|
||||
|
||||
func forceRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, transfer config.TransferPolicy) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Transfer: transfer,
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func defaultTransfer() config.TransferPolicy {
|
||||
return config.TransferPolicy{
|
||||
OnDestinationSame: config.TransferActionSkip,
|
||||
OnDestinationOlder: config.TransferActionReplace,
|
||||
OnDestinationNewer: config.TransferActionSkip,
|
||||
OnConflict: config.TransferActionFail,
|
||||
}
|
||||
}
|
||||
|
||||
func conflictReplaceTransfer() config.TransferPolicy {
|
||||
transfer := defaultTransfer()
|
||||
transfer.OnConflict = config.TransferActionReplace
|
||||
return transfer
|
||||
}
|
||||
|
||||
func newerReplaceTransfer() config.TransferPolicy {
|
||||
transfer := defaultTransfer()
|
||||
transfer.OnDestinationNewer = config.TransferActionReplace
|
||||
return transfer
|
||||
}
|
||||
@@ -29,6 +29,7 @@ func PlanOutputs(ctx context.Context, req Request) ([]Output, error) {
|
||||
Markdown: transform.MarkdownOptions{
|
||||
Mode: req.Transform.MarkdownToHTML.Mode,
|
||||
Input: req.Transform.MarkdownToHTML.Input,
|
||||
CssHref: req.Transform.MarkdownToHTML.CssHref,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -97,50 +98,10 @@ func rejectOutputCollisions(outputs []Output) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o Output) StateOutputFile() state.OutputFile {
|
||||
return state.OutputFile{
|
||||
Path: o.DestinationPath,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
}
|
||||
}
|
||||
|
||||
func (o Output) StateOutputProjection() state.OutputProjection {
|
||||
return state.OutputProjection{
|
||||
Path: o.DestinationPath,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
}
|
||||
}
|
||||
|
||||
func (o Output) ManagedPath() string {
|
||||
return o.DestinationPath
|
||||
}
|
||||
|
||||
func StateOutputFiles(outputs []Output) []state.OutputFile {
|
||||
files := make([]state.OutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
files = append(files, output.StateOutputFile())
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func StateOutputProjections(outputs []Output) []state.OutputProjection {
|
||||
projections := make([]state.OutputProjection, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
projections = append(projections, output.StateOutputProjection())
|
||||
}
|
||||
return projections
|
||||
}
|
||||
|
||||
func ManagedOutputPaths(outputs []Output) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
|
||||
@@ -12,46 +12,12 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
)
|
||||
|
||||
func TestOutputStateProjection(t *testing.T) {
|
||||
sourceOutput := Output{
|
||||
SourcePath: "report.md",
|
||||
DestinationPath: "report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
URL: "https://reports.example.com/report.md",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 123,
|
||||
}
|
||||
sourceState := sourceOutput.StateOutputFile()
|
||||
if sourceState.Path != "report.md" || sourceState.Kind != state.OutputKindSource || sourceState.SourcePath != "report.md" || sourceState.URL != sourceOutput.URL || sourceState.SHA256 != sourceOutput.SHA256 || sourceState.Size != sourceOutput.Size {
|
||||
t.Fatalf("source state output = %#v", sourceState)
|
||||
}
|
||||
|
||||
generatedOutput := Output{
|
||||
SourcePath: "report.md",
|
||||
DestinationPath: "report.html",
|
||||
Kind: state.OutputKindGenerated,
|
||||
Transform: transform.MarkdownToHTML,
|
||||
URL: "https://reports.example.com/report.html",
|
||||
SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
Size: 456,
|
||||
}
|
||||
generatedState := generatedOutput.StateOutputFile()
|
||||
if generatedState.Path != "report.html" || generatedState.Kind != state.OutputKindGenerated || generatedState.SourcePath != "report.md" || generatedState.Transform != transform.MarkdownToHTML || generatedState.URL != generatedOutput.URL || generatedState.SHA256 != generatedOutput.SHA256 || generatedState.Size != generatedOutput.Size {
|
||||
t.Fatalf("generated state output = %#v", generatedState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputSliceProjections(t *testing.T) {
|
||||
func TestManagedOutputPaths(t *testing.T) {
|
||||
outputs := []Output{
|
||||
{SourcePath: "report.md", DestinationPath: "report.md", Kind: state.OutputKindSource, SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Size: 1},
|
||||
{SourcePath: "report.md", DestinationPath: "report.html", Kind: state.OutputKindGenerated, Transform: transform.MarkdownToHTML, URL: "https://reports.example.com/report.html", SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Size: 2},
|
||||
}
|
||||
|
||||
stateOutputs := StateOutputFiles(outputs)
|
||||
if len(stateOutputs) != 2 || stateOutputs[1].Path != "report.html" || stateOutputs[1].Transform != transform.MarkdownToHTML || stateOutputs[1].URL != outputs[1].URL {
|
||||
t.Fatalf("state outputs = %#v", stateOutputs)
|
||||
}
|
||||
|
||||
paths := ManagedOutputPaths(outputs)
|
||||
if len(paths) != 2 || paths[0] != "report.md" || paths[1] != "report.html" {
|
||||
t.Fatalf("managed paths = %#v", paths)
|
||||
@@ -167,6 +133,7 @@ func TestPlanOutputsPassesMarkdownOptions(t *testing.T) {
|
||||
Enabled: true,
|
||||
Mode: config.TransformModeIndex,
|
||||
Input: "report.md",
|
||||
CssHref: "/assets/report.css",
|
||||
}},
|
||||
Transformers: testResolver{transform.MarkdownToHTML: transformer},
|
||||
})
|
||||
@@ -174,8 +141,8 @@ func TestPlanOutputsPassesMarkdownOptions(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("PlanOutputs() error = %v", err)
|
||||
}
|
||||
if transformer.request.Markdown.Mode != config.TransformModeIndex || transformer.request.Markdown.Input != "report.md" {
|
||||
t.Fatalf("markdown options = %#v, want index/report.md", transformer.request.Markdown)
|
||||
if transformer.request.Markdown.Mode != config.TransformModeIndex || transformer.request.Markdown.Input != "report.md" || transformer.request.Markdown.CssHref != "/assets/report.css" {
|
||||
t.Fatalf("markdown options = %#v, want index/report.md with css href", transformer.request.Markdown)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,12 +160,6 @@ func TestBuildRejectsHTMLWithoutTransform(t *testing.T) {
|
||||
DestinationBundlePath: "",
|
||||
SourceBundle: sourceBundle,
|
||||
Publish: config.PublishPolicy{HTML: true},
|
||||
Transfer: config.TransferPolicy{
|
||||
OnDestinationSame: config.TransferActionSkip,
|
||||
OnDestinationOlder: config.TransferActionReplace,
|
||||
OnDestinationNewer: config.TransferActionSkip,
|
||||
OnConflict: config.TransferActionFail,
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want missing transform error")
|
||||
|
||||
@@ -3,6 +3,7 @@ package publish
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
@@ -15,12 +16,12 @@ type Action string
|
||||
|
||||
const (
|
||||
ActionPublishNew Action = "publish_new"
|
||||
ActionReplaceOlder Action = "replace_older"
|
||||
ActionSkipSame Action = "skip_same"
|
||||
ActionSkipDestinationNewer Action = "skip_destination_newer"
|
||||
ActionFailConflict Action = "fail_conflict"
|
||||
ActionFailUnmanaged Action = "fail_unmanaged"
|
||||
ActionForceReplace Action = "force_replace"
|
||||
ActionUpsertAdditive Action = "upsert_additive"
|
||||
ActionReplaceCatalog Action = "replace_catalog"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
@@ -34,18 +35,28 @@ type Request struct {
|
||||
Publish config.PublishPolicy
|
||||
Transform config.Transform
|
||||
Links *config.Links
|
||||
State config.StatePolicy
|
||||
Reconciliation config.ReconciliationPolicy
|
||||
Workflow string
|
||||
Transformers TransformerResolver
|
||||
Transfer config.TransferPolicy
|
||||
DistributorVersion string
|
||||
Force bool
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type TransformerResolver interface {
|
||||
Get(name string) (transform.Transformer, bool)
|
||||
}
|
||||
|
||||
type Output struct {
|
||||
SourcePath string
|
||||
DestinationPath string
|
||||
Kind string
|
||||
Transform string
|
||||
URL string
|
||||
Data []byte
|
||||
SHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type Plan struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
@@ -57,27 +68,24 @@ type Plan struct {
|
||||
Reason string
|
||||
Force bool
|
||||
PrimaryURL string
|
||||
StateMode string
|
||||
Workflow string
|
||||
OwnerScope state.OwnerScope
|
||||
Reconciliation config.ReconciliationPolicy
|
||||
Outputs []Output
|
||||
ExistingState *state.DistributorState
|
||||
ExistingSharedRoot *state.SharedRootState
|
||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
||||
RetainedOwnerOutputs []state.SharedRootOutputFile
|
||||
OwnerOutputsToDelete []state.SharedRootOutputFile
|
||||
OwnerOutputsToWrite []Output
|
||||
ExistingCatalog *state.CatalogState
|
||||
SupersededLegacy *state.SupersededLegacyState
|
||||
CatalogOutputsToWrite []state.CatalogOutputFile
|
||||
CatalogOutputsToRetain []state.CatalogOutputFile
|
||||
CatalogOutputsToDelete []state.CatalogOutputFile
|
||||
ClearDestinationRoot bool
|
||||
}
|
||||
|
||||
type Output struct {
|
||||
SourcePath string
|
||||
DestinationPath string
|
||||
Kind string
|
||||
Transform string
|
||||
URL string
|
||||
Data []byte
|
||||
SHA256 string
|
||||
Size int64
|
||||
type catalogPlanDetails struct {
|
||||
Action Action
|
||||
Reason string
|
||||
CatalogOutputsToWrite []state.CatalogOutputFile
|
||||
CatalogOutputsToRetain []state.CatalogOutputFile
|
||||
CatalogOutputsToDelete []state.CatalogOutputFile
|
||||
ClearDestinationRoot bool
|
||||
}
|
||||
|
||||
func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
@@ -96,10 +104,9 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
comparison := compareDestination(req, status)
|
||||
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
|
||||
reconciliation := normalizeReconciliation(req.Reconciliation)
|
||||
stateMode := normalizeState(req.State).Mode
|
||||
workflow := normalizeWorkflow(req.Workflow)
|
||||
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
now := requestTime(req)
|
||||
plan := Plan{
|
||||
PipelineID: req.PipelineID,
|
||||
DestinationID: req.DestinationID,
|
||||
@@ -107,32 +114,53 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
BundlePath: req.SourceBundle.RootRelativePath,
|
||||
DestinationBundlePath: req.DestinationBundlePath,
|
||||
PathMapping: req.PathMapping,
|
||||
Action: action,
|
||||
Reason: reason,
|
||||
Force: action == ActionForceReplace,
|
||||
PrimaryURL: primaryURL,
|
||||
StateMode: stateMode,
|
||||
OwnerScope: state.CurrentOwnerScope(req.PipelineID, req.DestinationID),
|
||||
Reconciliation: reconciliation,
|
||||
Workflow: workflow,
|
||||
OwnerScope: scope,
|
||||
Outputs: outputs,
|
||||
ExistingState: status.State,
|
||||
ExistingSharedRoot: status.SharedRoot,
|
||||
ExistingCatalog: status.Catalog,
|
||||
SupersededLegacy: status.SupersededLegacy,
|
||||
}
|
||||
if status.StateErr != nil {
|
||||
plan.Reason = status.StateErr.Error()
|
||||
if req.Force {
|
||||
plan.Action = ActionForceReplace
|
||||
plan.Force = true
|
||||
plan.ClearDestinationRoot = true
|
||||
plan.CatalogOutputsToWrite = catalogOutputsForPlan(req, outputs, nil, scope, now)
|
||||
return plan, nil
|
||||
}
|
||||
plan.Action = ActionFailConflict
|
||||
return plan, fmt.Errorf("%s: %s", plan.Action, plan.Reason)
|
||||
}
|
||||
|
||||
var details catalogPlanDetails
|
||||
switch {
|
||||
case status.Catalog != nil:
|
||||
details, err = planExistingCatalog(ctx, req, *status.Catalog, outputs, workflow, scope, now)
|
||||
case status.SupersededLegacy != nil:
|
||||
details = planSupersededLegacy(req, outputs, workflow, scope, now)
|
||||
default:
|
||||
details, err = planWithoutCatalog(ctx, req, outputs, workflow, scope, now, status.HasContents)
|
||||
}
|
||||
plan.Action = details.Action
|
||||
plan.Reason = details.Reason
|
||||
plan.CatalogOutputsToWrite = details.CatalogOutputsToWrite
|
||||
plan.CatalogOutputsToRetain = details.CatalogOutputsToRetain
|
||||
plan.CatalogOutputsToDelete = details.CatalogOutputsToDelete
|
||||
plan.ClearDestinationRoot = details.ClearDestinationRoot
|
||||
if err != nil && req.Force && forceCanReplace(details.Action) {
|
||||
plan.Action = ActionForceReplace
|
||||
plan.Force = true
|
||||
plan.ClearDestinationRoot = true
|
||||
plan.CatalogOutputsToWrite = catalogOutputsForPlan(req, outputs, nil, scope, now)
|
||||
plan.CatalogOutputsToRetain = nil
|
||||
plan.CatalogOutputsToDelete = nil
|
||||
return plan, nil
|
||||
}
|
||||
if stateMode == config.StateModeSharedRoot {
|
||||
sharedDetails, err := planSharedRootOwner(ctx, req, status, action, reconciliation, outputs)
|
||||
plan.OtherOwnerOutputs = sharedDetails.OtherOwnerOutputs
|
||||
plan.RetainedOwnerOutputs = sharedDetails.RetainedOwnerOutputs
|
||||
plan.OwnerOutputsToDelete = sharedDetails.OwnerOutputsToDelete
|
||||
plan.OwnerOutputsToWrite = sharedDetails.OwnerOutputsToWrite
|
||||
if err != nil {
|
||||
plan.Action = sharedDetails.Action
|
||||
plan.Reason = sharedDetails.Reason
|
||||
return plan, err
|
||||
}
|
||||
}
|
||||
if action == ActionFailConflict || action == ActionFailUnmanaged {
|
||||
return plan, fmt.Errorf("%s: %s", action, reason)
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
@@ -152,139 +180,180 @@ func validateRequest(req Request) error {
|
||||
if err := config.ValidatePublishTransformPolicy(req.Publish, req.Transform); err != nil {
|
||||
return fmt.Errorf("publish/transform policy: %w", err)
|
||||
}
|
||||
switch normalizeReconciliation(req.Reconciliation).Mode {
|
||||
case config.ReconciliationModeReplace, config.ReconciliationModeMerge:
|
||||
switch normalizeWorkflow(req.Workflow) {
|
||||
case config.WorkflowAdditive, config.WorkflowReplacement:
|
||||
default:
|
||||
return fmt.Errorf("reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
switch normalizeState(req.State).Mode {
|
||||
case config.StateModeSingleOwner, config.StateModeSharedRoot:
|
||||
default:
|
||||
return fmt.Errorf("state.mode must be %s or %s", config.StateModeSingleOwner, config.StateModeSharedRoot)
|
||||
return fmt.Errorf("destination.workflow must be %s or %s", config.WorkflowAdditive, config.WorkflowReplacement)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.ReconciliationModeReplace
|
||||
func normalizeWorkflow(workflow string) string {
|
||||
if workflow == "" {
|
||||
return config.WorkflowAdditive
|
||||
}
|
||||
return policy
|
||||
return workflow
|
||||
}
|
||||
|
||||
func normalizeState(policy config.StatePolicy) config.StatePolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.StateModeSingleOwner
|
||||
func requestTime(req Request) time.Time {
|
||||
if req.Now.IsZero() {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return policy
|
||||
return req.Now.UTC()
|
||||
}
|
||||
|
||||
func compareDestination(req Request, status state.DestinationStatus) state.Comparison {
|
||||
if normalizeState(req.State).Mode == config.StateModeSharedRoot {
|
||||
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
comparison := state.CompareSharedRootOwner(req.SourceBundle.Manifest, scope, status)
|
||||
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict {
|
||||
return comparison
|
||||
func planExistingCatalog(ctx context.Context, req Request, catalog state.CatalogState, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) (catalogPlanDetails, error) {
|
||||
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, catalog.Outputs, outputs); err != nil {
|
||||
return catalogPlanDetails{
|
||||
Action: ActionFailUnmanaged,
|
||||
Reason: err.Error(),
|
||||
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||
}
|
||||
destinationManifest, ok := sharedRootComparisonManifest(status, scope)
|
||||
if !ok {
|
||||
return comparison
|
||||
planned := outputPathSet(outputs)
|
||||
details := catalogPlanDetails{
|
||||
Action: actionForWorkflow(workflow),
|
||||
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, catalog.Outputs, scope, now),
|
||||
}
|
||||
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
|
||||
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
|
||||
}
|
||||
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
|
||||
return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"}
|
||||
}
|
||||
return comparison
|
||||
}
|
||||
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
|
||||
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict || status.State == nil {
|
||||
return comparison
|
||||
}
|
||||
destinationManifest := status.State.Source.Manifest
|
||||
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
|
||||
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
|
||||
}
|
||||
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
|
||||
return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"}
|
||||
}
|
||||
return comparison
|
||||
}
|
||||
|
||||
func sharedRootComparisonManifest(status state.DestinationStatus, scope state.OwnerScope) (bundle.Manifest, bool) {
|
||||
if status.SharedRoot != nil {
|
||||
return status.SharedRoot.SourceManifest(scope)
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
return status.State.Source.Manifest, true
|
||||
}
|
||||
return bundle.Manifest{}, false
|
||||
}
|
||||
|
||||
type sharedRootPlanDetails struct {
|
||||
Action Action
|
||||
Reason string
|
||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
||||
RetainedOwnerOutputs []state.SharedRootOutputFile
|
||||
OwnerOutputsToDelete []state.SharedRootOutputFile
|
||||
OwnerOutputsToWrite []Output
|
||||
}
|
||||
|
||||
func planSharedRootOwner(ctx context.Context, req Request, status state.DestinationStatus, action Action, reconciliation config.ReconciliationPolicy, outputs []Output) (sharedRootPlanDetails, error) {
|
||||
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
details := sharedRootPlanDetails{Action: action}
|
||||
if !isWriteAction(action) {
|
||||
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
|
||||
return details, nil
|
||||
}
|
||||
|
||||
plannedPaths := outputPaths(outputs)
|
||||
if action == ActionForceReplace {
|
||||
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
|
||||
return details, nil
|
||||
}
|
||||
if conflict, ok := sharedRootPathOwnershipConflict(status, scope, plannedPaths); ok {
|
||||
reason := fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
details.Action = ActionFailConflict
|
||||
details.Reason = reason
|
||||
return details, fmt.Errorf("%s: %s", ActionFailConflict, reason)
|
||||
}
|
||||
if err := rejectSharedRootUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, status, scope, plannedPaths); err != nil {
|
||||
details.Action = ActionFailUnmanaged
|
||||
details.Reason = err.Error()
|
||||
return details, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||
}
|
||||
|
||||
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
|
||||
ownerOutputs := currentOwnerOutputs(status, scope)
|
||||
planned := make(map[string]struct{}, len(plannedPaths))
|
||||
for _, path := range plannedPaths {
|
||||
planned[path] = struct{}{}
|
||||
}
|
||||
for _, output := range ownerOutputs {
|
||||
allPlannedOutputsMatch := catalogContainsMatchingOutputs(req, catalog.Outputs, outputs, scope)
|
||||
for _, output := range catalog.Outputs {
|
||||
if _, exists := planned[output.Path]; exists {
|
||||
continue
|
||||
}
|
||||
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeReplace {
|
||||
details.OwnerOutputsToDelete = append(details.OwnerOutputsToDelete, output)
|
||||
if workflow == config.WorkflowReplacement && output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
|
||||
details.CatalogOutputsToDelete = append(details.CatalogOutputsToDelete, output)
|
||||
continue
|
||||
}
|
||||
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
details.RetainedOwnerOutputs = append(details.RetainedOwnerOutputs, output)
|
||||
details.CatalogOutputsToRetain = append(details.CatalogOutputsToRetain, output)
|
||||
}
|
||||
if allPlannedOutputsMatch && (workflow == config.WorkflowAdditive || len(details.CatalogOutputsToDelete) == 0) {
|
||||
details.Action = ActionSkipSame
|
||||
details.CatalogOutputsToWrite = nil
|
||||
details.CatalogOutputsToDelete = nil
|
||||
}
|
||||
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
|
||||
return details, nil
|
||||
}
|
||||
|
||||
func isWriteAction(action Action) bool {
|
||||
switch action {
|
||||
case ActionPublishNew, ActionReplaceOlder, ActionForceReplace:
|
||||
return true
|
||||
default:
|
||||
func catalogContainsMatchingOutputs(req Request, existing []state.CatalogOutputFile, outputs []Output, scope state.OwnerScope) bool {
|
||||
for _, output := range outputs {
|
||||
catalogOutput, ok := state.FindCatalogOutputByPath(existing, output.DestinationPath)
|
||||
if !ok || !catalogOutputMatchesPlan(req, catalogOutput, output, scope) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func catalogOutputMatchesPlan(req Request, catalogOutput state.CatalogOutputFile, output Output, scope state.OwnerScope) bool {
|
||||
if catalogOutput.PipelineID != scope.PipelineID ||
|
||||
catalogOutput.DestinationID != scope.DestinationID ||
|
||||
catalogOutput.Source.ID != req.SourceBundle.Manifest.ID ||
|
||||
catalogOutput.Source.Digest != req.SourceBundle.Manifest.Digest ||
|
||||
!catalogOutput.Source.Created.Equal(req.SourceBundle.Manifest.Created) ||
|
||||
catalogOutput.Path != output.DestinationPath ||
|
||||
catalogOutput.Kind != output.Kind ||
|
||||
catalogOutput.URL != output.URL ||
|
||||
catalogOutput.SHA256 != output.SHA256 ||
|
||||
catalogOutput.Size != output.Size {
|
||||
return false
|
||||
}
|
||||
if output.Kind == state.OutputKindGenerated {
|
||||
return catalogOutput.SourcePath == output.SourcePath && catalogOutput.Transform == output.Transform
|
||||
}
|
||||
return catalogOutput.SourcePath == "" && catalogOutput.Transform == ""
|
||||
}
|
||||
|
||||
func planSupersededLegacy(req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) catalogPlanDetails {
|
||||
details := catalogPlanDetails{
|
||||
Action: actionForWorkflow(workflow),
|
||||
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, nil, scope, now),
|
||||
}
|
||||
if workflow == config.WorkflowReplacement {
|
||||
details.ClearDestinationRoot = true
|
||||
}
|
||||
return details
|
||||
}
|
||||
|
||||
func planWithoutCatalog(ctx context.Context, req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time, hasContents bool) (catalogPlanDetails, error) {
|
||||
if hasContents {
|
||||
err := fmt.Errorf("destination has content but no distributor state")
|
||||
return catalogPlanDetails{
|
||||
Action: ActionFailUnmanaged,
|
||||
Reason: err.Error(),
|
||||
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||
}
|
||||
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, nil, outputs); err != nil {
|
||||
return catalogPlanDetails{
|
||||
Action: ActionFailUnmanaged,
|
||||
Reason: err.Error(),
|
||||
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||
}
|
||||
return catalogPlanDetails{
|
||||
Action: ActionPublishNew,
|
||||
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, nil, scope, now),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func forceCanReplace(action Action) bool {
|
||||
return action == ActionFailUnmanaged || action == ActionFailConflict
|
||||
}
|
||||
|
||||
func actionForWorkflow(workflow string) Action {
|
||||
if workflow == config.WorkflowReplacement {
|
||||
return ActionReplaceCatalog
|
||||
}
|
||||
return ActionUpsertAdditive
|
||||
}
|
||||
|
||||
func catalogOutputsForPlan(req Request, outputs []Output, existing []state.CatalogOutputFile, scope state.OwnerScope, now time.Time) []state.CatalogOutputFile {
|
||||
files := make([]state.CatalogOutputFile, 0, len(outputs))
|
||||
source := state.CatalogSourceIdentity{
|
||||
ID: req.SourceBundle.Manifest.ID,
|
||||
Digest: req.SourceBundle.Manifest.Digest,
|
||||
Created: req.SourceBundle.Manifest.Created,
|
||||
}
|
||||
for _, output := range outputs {
|
||||
createdAt := now
|
||||
if existingOutput, ok := state.FindCatalogOutputByPath(existing, output.DestinationPath); ok {
|
||||
createdAt = existingOutput.CreatedAt
|
||||
}
|
||||
file := state.CatalogOutputFile{
|
||||
Path: output.DestinationPath,
|
||||
PipelineID: scope.PipelineID,
|
||||
DestinationID: scope.DestinationID,
|
||||
Source: source,
|
||||
Kind: output.Kind,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if output.Kind == state.OutputKindGenerated {
|
||||
file.SourcePath = output.SourcePath
|
||||
file.Transform = output.Transform
|
||||
}
|
||||
files = append(files, file)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func rejectCatalogUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, existing []state.CatalogOutputFile, outputs []Output) error {
|
||||
managed := catalogOutputPathSet(existing)
|
||||
for _, output := range outputs {
|
||||
if _, exists := managed[output.DestinationPath]; exists {
|
||||
continue
|
||||
}
|
||||
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return fmt.Errorf("destination output path %s exists but is not managed by catalog state", storage.DisplayPath(output.DestinationPath))
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func outputPaths(outputs []Output) []string {
|
||||
@@ -295,131 +364,18 @@ func outputPaths(outputs []Output) []string {
|
||||
return paths
|
||||
}
|
||||
|
||||
func sharedRootPathOwnershipConflict(status state.DestinationStatus, scope state.OwnerScope, paths []string) (state.PathOwnershipConflict, bool) {
|
||||
if status.SharedRoot != nil {
|
||||
return status.SharedRoot.PathOwnershipConflict(scope, paths)
|
||||
func outputPathSet(outputs []Output) map[string]struct{} {
|
||||
paths := make(map[string]struct{}, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths[output.DestinationPath] = struct{}{}
|
||||
}
|
||||
return state.PathOwnershipConflict{}, false
|
||||
return paths
|
||||
}
|
||||
|
||||
func rejectSharedRootUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, status state.DestinationStatus, scope state.OwnerScope, paths []string) error {
|
||||
for _, path := range paths {
|
||||
if pathManagedBySharedRootStatus(status, scope, path) {
|
||||
continue
|
||||
}
|
||||
destinationPath, err := storage.Join(bundlePath, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return fmt.Errorf("destination output path %s exists but is not managed by destination state", storage.DisplayPath(path))
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pathManagedBySharedRootStatus(status state.DestinationStatus, scope state.OwnerScope, path string) bool {
|
||||
if status.SharedRoot != nil {
|
||||
_, exists := status.SharedRoot.OutputOwner(path)
|
||||
return exists
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
_, exists := state.FindOutputByPath(status.State.Outputs, path)
|
||||
return exists
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func otherOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
if status.SharedRoot == nil {
|
||||
return nil
|
||||
}
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
|
||||
for _, output := range status.SharedRoot.Outputs {
|
||||
if output.Owner != scope {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
|
||||
func currentOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
if status.SharedRoot != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
|
||||
for _, output := range status.SharedRoot.Outputs {
|
||||
if output.Owner == scope {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.State.Outputs))
|
||||
for _, output := range status.State.Outputs {
|
||||
outputs = append(outputs, state.SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: scope,
|
||||
SourceID: status.State.Source.Manifest.ID,
|
||||
SourceDigest: status.State.Source.Manifest.Digest,
|
||||
SourceCreated: status.State.Source.Manifest.Created,
|
||||
CreatedAt: output.CreatedAt,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
|
||||
switch comparison.Outcome {
|
||||
case state.OutcomeDestinationAbsent:
|
||||
return ActionPublishNew, comparison.Reason
|
||||
case state.OutcomeDestinationUnmanaged:
|
||||
if force {
|
||||
return ActionForceReplace, "forced replacement of unmanaged destination content"
|
||||
}
|
||||
return ActionFailUnmanaged, comparison.Reason
|
||||
case state.OutcomeInvalidState:
|
||||
return ActionFailConflict, comparison.Reason
|
||||
case state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
|
||||
if transfer.OnConflict == config.TransferActionReplace {
|
||||
if force {
|
||||
return ActionForceReplace, "forced replacement of conflicting destination state: " + comparison.Reason
|
||||
}
|
||||
return ActionFailConflict, "destination conflict replacement requires --force"
|
||||
}
|
||||
return ActionFailConflict, comparison.Reason
|
||||
case state.OutcomeSameSource:
|
||||
if transfer.OnDestinationSame == config.TransferActionFail {
|
||||
return ActionFailConflict, "destination matches source and transfer policy requires failure"
|
||||
}
|
||||
return ActionSkipSame, comparison.Reason
|
||||
case state.OutcomeDestinationOlder:
|
||||
if transfer.OnDestinationOlder == config.TransferActionFail {
|
||||
return ActionFailConflict, "destination is older and transfer policy requires failure"
|
||||
}
|
||||
return ActionReplaceOlder, comparison.Reason
|
||||
case state.OutcomeDestinationNewer:
|
||||
if transfer.OnDestinationNewer == config.TransferActionReplace {
|
||||
if force {
|
||||
return ActionForceReplace, "forced replacement of newer destination state"
|
||||
}
|
||||
return ActionFailConflict, "destination is newer and replacement requires --force"
|
||||
}
|
||||
if transfer.OnDestinationNewer == config.TransferActionFail {
|
||||
return ActionFailConflict, "destination is newer and transfer policy requires failure"
|
||||
}
|
||||
return ActionSkipDestinationNewer, comparison.Reason
|
||||
default:
|
||||
return ActionFailConflict, "unsupported comparison outcome"
|
||||
func catalogOutputPathSet(outputs []state.CatalogOutputFile) map[string]struct{} {
|
||||
paths := make(map[string]struct{}, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths[output.Path] = struct{}{}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
624
internal/publish/plan_test.go
Normal file
624
internal/publish/plan_test.go
Normal file
@@ -0,0 +1,624 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
)
|
||||
|
||||
var (
|
||||
planCreatedAt = time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
planUpdatedAt = time.Date(2026, 6, 1, 9, 30, 0, 0, time.UTC)
|
||||
)
|
||||
|
||||
func TestBuildAdditivePublishesCatalogOutputs(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionPublishNew || plan.Workflow != config.WorkflowAdditive {
|
||||
t.Fatalf("plan action=%s workflow=%s, want publish_new additive", plan.Action, plan.Workflow)
|
||||
}
|
||||
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
|
||||
}
|
||||
for _, output := range plan.CatalogOutputsToWrite {
|
||||
if output.PipelineID != "reports" || output.DestinationID != "archive" {
|
||||
t.Fatalf("catalog output owner = %s/%s", output.PipelineID, output.DestinationID)
|
||||
}
|
||||
if output.Source.ID != req.SourceBundle.Manifest.ID || output.Source.Digest != req.SourceBundle.Manifest.Digest || !output.Source.Created.Equal(req.SourceBundle.Manifest.Created) {
|
||||
t.Fatalf("catalog output source = %#v", output.Source)
|
||||
}
|
||||
if output.Kind == state.OutputKindSource && output.SourcePath != "" {
|
||||
t.Fatalf("source catalog output source_path = %q, want empty", output.SourcePath)
|
||||
}
|
||||
if !output.CreatedAt.Equal(planUpdatedAt) || !output.UpdatedAt.Equal(planUpdatedAt) {
|
||||
t.Fatalf("catalog output times = %s/%s", output.CreatedAt, output.UpdatedAt)
|
||||
}
|
||||
}
|
||||
if _, err := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(err) {
|
||||
t.Fatalf("destination state stat error = %v, want missing", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAdditiveOverwritesManagedAndRetainsOthers(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionUpsertAdditive {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionUpsertAdditive)
|
||||
}
|
||||
if len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("delete outputs = %#v, want none", plan.CatalogOutputsToDelete)
|
||||
}
|
||||
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "old.txt" {
|
||||
t.Fatalf("retained outputs = %#v, want old.txt", plan.CatalogOutputsToRetain)
|
||||
}
|
||||
written, ok := state.FindCatalogOutputByPath(plan.CatalogOutputsToWrite, "report.md")
|
||||
if !ok {
|
||||
t.Fatalf("written outputs = %#v, want report.md", plan.CatalogOutputsToWrite)
|
||||
}
|
||||
if !written.CreatedAt.Equal(planCreatedAt) || !written.UpdatedAt.Equal(planUpdatedAt) {
|
||||
t.Fatalf("report.md times = %s/%s, want created preserved and updated now", written.CreatedAt, written.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReplacementDeletesCurrentOwnerAndRetainsOtherOwners(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
|
||||
catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceCatalog {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceCatalog)
|
||||
}
|
||||
if len(plan.CatalogOutputsToDelete) != 1 || plan.CatalogOutputsToDelete[0].Path != "stale.txt" {
|
||||
t.Fatalf("delete outputs = %#v, want stale.txt", plan.CatalogOutputsToDelete)
|
||||
}
|
||||
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "shared.txt" {
|
||||
t.Fatalf("retained outputs = %#v, want shared.txt", plan.CatalogOutputsToRetain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAdditiveSkipsMatchingCatalogOutputs(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
existing := matchingCatalogForRequest(t, req)
|
||||
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt))
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionSkipSame {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionSkipSame)
|
||||
}
|
||||
if len(plan.CatalogOutputsToWrite) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("catalog write=%d delete=%d, want no writes or deletes", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToDelete))
|
||||
}
|
||||
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "old.txt" {
|
||||
t.Fatalf("retained outputs = %#v, want old.txt", plan.CatalogOutputsToRetain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReplacementSkipsMatchingCatalogOutputs(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
existing := matchingCatalogForRequest(t, req)
|
||||
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt))
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionSkipSame {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionSkipSame)
|
||||
}
|
||||
if len(plan.CatalogOutputsToWrite) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("catalog write=%d delete=%d, want no writes or deletes", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToDelete))
|
||||
}
|
||||
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "shared.txt" {
|
||||
t.Fatalf("retained outputs = %#v, want shared.txt", plan.CatalogOutputsToRetain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReplacementDoesNotSkipWhenCurrentOwnerOutputWouldBeDeleted(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
existing := matchingCatalogForRequest(t, req)
|
||||
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt))
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceCatalog {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceCatalog)
|
||||
}
|
||||
if len(plan.CatalogOutputsToDelete) != 1 || plan.CatalogOutputsToDelete[0].Path != "stale.txt" {
|
||||
t.Fatalf("delete outputs = %#v, want stale.txt", plan.CatalogOutputsToDelete)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDoesNotSkipWhenCatalogMetadataDiffers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*Request)
|
||||
mutate func(*state.CatalogState)
|
||||
}{
|
||||
{
|
||||
name: "pipeline id",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].PipelineID = "other"
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "destination id",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].DestinationID = "web"
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "source id",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].Source.ID = "reports.other"
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "source digest",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].Source.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "source created",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].Source.Created = catalog.Outputs[0].Source.Created.Add(time.Second)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "output kind",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].Kind = state.OutputKindGenerated
|
||||
catalog.Outputs[0].SourcePath = "report.md"
|
||||
catalog.Outputs[0].Transform = transform.MarkdownToHTML
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "output digest",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].SHA256 = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "output size",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].Size++
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "url",
|
||||
configure: func(req *Request) {
|
||||
req.Links = &config.Links{BaseURL: "https://reports.example.com/archive", Primary: config.LinkPrimaryAuto}
|
||||
},
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].URL = "https://reports.example.com/archive/old-report.md"
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
if tt.configure != nil {
|
||||
tt.configure(&req)
|
||||
}
|
||||
existing := matchingCatalogForRequest(t, req)
|
||||
tt.mutate(&existing)
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action == ActionSkipSame {
|
||||
t.Fatalf("plan action = %s, want write action after metadata change", plan.Action)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDoesNotSkipWhenGeneratedCatalogMetadataDiffers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*state.CatalogState)
|
||||
}{
|
||||
{
|
||||
name: "source path",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].SourcePath = "summary.md"
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "transform",
|
||||
mutate: func(catalog *state.CatalogState) {
|
||||
catalog.Outputs[0].Transform = "markdown_to_html_index"
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, destinationBackend, req := catalogGeneratedPlanRequest(t, config.WorkflowAdditive)
|
||||
existing := matchingCatalogForRequest(t, req)
|
||||
tt.mutate(&existing)
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action == ActionSkipSame {
|
||||
t.Fatalf("plan action = %s, want write action after generated metadata change", plan.Action)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTransfersManagedPathOwnership(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
existing := baseCatalog(req)
|
||||
existing.Outputs = []state.CatalogOutputFile{
|
||||
catalogOutput(req, "reports", "web", "report.md", state.OutputKindSource, planCreatedAt),
|
||||
}
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
written, ok := state.FindCatalogOutputByPath(plan.CatalogOutputsToWrite, "report.md")
|
||||
if !ok {
|
||||
t.Fatalf("written outputs = %#v, want report.md", plan.CatalogOutputsToWrite)
|
||||
}
|
||||
if written.PipelineID != "reports" || written.DestinationID != "archive" {
|
||||
t.Fatalf("written owner = %s/%s, want reports/archive", written.PipelineID, written.DestinationID)
|
||||
}
|
||||
if !written.CreatedAt.Equal(planCreatedAt) || !written.UpdatedAt.Equal(planUpdatedAt) {
|
||||
t.Fatalf("written times = %s/%s", written.CreatedAt, written.UpdatedAt)
|
||||
}
|
||||
if len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("retain=%#v delete=%#v, want no old record for overwritten path", plan.CatalogOutputsToRetain, plan.CatalogOutputsToDelete)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsUnmanagedPlannedPathCollision(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
writeCatalogState(t, destinationBackend, "", baseCatalog(req))
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want unmanaged collision")
|
||||
}
|
||||
if plan.Action != ActionFailUnmanaged || !strings.Contains(err.Error(), "not managed by catalog state") {
|
||||
t.Fatalf("plan action=%s error=%v, want unmanaged catalog collision", plan.Action, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildForceReplacesUnmanagedPlannedPathCollision(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
req.Force = true
|
||||
existing := baseCatalog(req)
|
||||
writeCatalogState(t, destinationBackend, "", existing)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
|
||||
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
|
||||
}
|
||||
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsNoStateNonEmptyDestinationWithoutForce(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "unmanaged")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want unmanaged destination")
|
||||
}
|
||||
if plan.Action != ActionFailUnmanaged || !strings.Contains(err.Error(), "destination has content but no distributor state") {
|
||||
t.Fatalf("plan action=%s error=%v, want unmanaged destination", plan.Action, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildForceReplacesNoStateNonEmptyDestination(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
req.Force = true
|
||||
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "unmanaged")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
|
||||
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
|
||||
}
|
||||
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
legacyState := legacyStateDocument(2)
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != 2 {
|
||||
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
|
||||
}
|
||||
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
|
||||
t.Fatalf("plan action=%s clear=%t, want additive overwrite without clear", plan.Action, plan.ClearDestinationRoot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||
legacyState := legacyStateDocument(2)
|
||||
writeJSONState(t, destinationBackend, "", legacyState)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceCatalog || !plan.ClearDestinationRoot {
|
||||
t.Fatalf("plan action=%s clear=%t, want replacement clear", plan.Action, plan.ClearDestinationRoot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRejectsInvalidOrFutureState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
}{
|
||||
{name: "invalid json", data: `{"schema_version":`},
|
||||
{name: "future schema", data: `{"schema_version":99}`},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
testutil.WriteFakeFile(t, destinationBackend, storage.StateFileName, tt.data)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want conflict")
|
||||
}
|
||||
if plan.Action != ActionFailConflict {
|
||||
t.Fatalf("plan action = %s, want %s", plan.Action, ActionFailConflict)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildForceReplacesInvalidOrFutureState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
}{
|
||||
{name: "invalid json", data: `{"schema_version":`},
|
||||
{name: "future schema", data: `{"schema_version":99}`},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||
req.Force = true
|
||||
testutil.WriteFakeFile(t, destinationBackend, storage.StateFileName, tt.data)
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
|
||||
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
|
||||
}
|
||||
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildForceDoesNotChangeValidCatalogActions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
workflow string
|
||||
want Action
|
||||
}{
|
||||
{name: "additive", workflow: config.WorkflowAdditive, want: ActionUpsertAdditive},
|
||||
{name: "replacement", workflow: config.WorkflowReplacement, want: ActionReplaceCatalog},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, destinationBackend, req := catalogPlanRequest(t, tt.workflow)
|
||||
req.Force = true
|
||||
writeCatalogState(t, destinationBackend, "", baseCatalog(req))
|
||||
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != tt.want || plan.Force || plan.ClearDestinationRoot {
|
||||
t.Fatalf("plan action=%s force=%t clear=%t, want %s without forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRequestRejectsInvalidWorkflow(t *testing.T) {
|
||||
sourceBackend, destinationBackend, req := catalogPlanRequest(t, "append")
|
||||
req.SourceBackend = sourceBackend
|
||||
req.DestinationBackend = destinationBackend
|
||||
|
||||
err := validateRequest(req)
|
||||
if err == nil {
|
||||
t.Fatal("validateRequest() error = nil, want invalid workflow")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "destination.workflow") {
|
||||
t.Fatalf("validateRequest() error = %v, want workflow context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func catalogPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Backend, Request) {
|
||||
t.Helper()
|
||||
sourceBackend := fake.New()
|
||||
destinationBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{})
|
||||
return sourceBackend, destinationBackend, Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: "",
|
||||
PathMapping: config.PathMappingPreserveRelative,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Workflow: workflow,
|
||||
DistributorVersion: "test",
|
||||
Now: planUpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func catalogGeneratedPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Backend, Request) {
|
||||
t.Helper()
|
||||
sourceBackend, destinationBackend, req := catalogPlanRequest(t, workflow)
|
||||
data := []byte("<p>Generated</p>\n")
|
||||
req.Publish = config.PublishPolicy{HTML: true}
|
||||
req.Transform = config.Transform{MarkdownToHTML: &config.MarkdownToHTML{
|
||||
Enabled: true,
|
||||
Mode: config.TransformModeSidecar,
|
||||
}}
|
||||
req.Transformers = testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
|
||||
Path: "report.html",
|
||||
SourcePath: "report.md",
|
||||
Transform: transform.MarkdownToHTML,
|
||||
Data: data,
|
||||
SHA256: bundle.FileDigest(data),
|
||||
Size: int64(len(data)),
|
||||
}}}}
|
||||
return sourceBackend, destinationBackend, req
|
||||
}
|
||||
|
||||
func matchingCatalogForRequest(t *testing.T, req Request) state.CatalogState {
|
||||
t.Helper()
|
||||
outputs, err := PlanOutputs(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("PlanOutputs() error = %v", err)
|
||||
}
|
||||
outputs, _, err = PlanLinks(req, outputs)
|
||||
if err != nil {
|
||||
t.Fatalf("PlanLinks() error = %v", err)
|
||||
}
|
||||
catalog := baseCatalog(req)
|
||||
catalog.Outputs = catalogOutputsForPlan(req, outputs, nil, state.CurrentOwnerScope(req.PipelineID, req.DestinationID), planCreatedAt)
|
||||
return catalog
|
||||
}
|
||||
|
||||
func baseCatalog(req Request) state.CatalogState {
|
||||
return state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: "previous",
|
||||
CreatedAt: planCreatedAt,
|
||||
UpdatedAt: planCreatedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: []state.CatalogOutputFile{},
|
||||
}
|
||||
}
|
||||
|
||||
func catalogOutput(req Request, pipelineID, destinationID, path, kind string, createdAt time.Time) state.CatalogOutputFile {
|
||||
sourceSHA := req.SourceBundle.Manifest.Files[0].SHA256
|
||||
sourceSize := req.SourceBundle.Manifest.Files[0].Size
|
||||
for _, file := range req.SourceBundle.Manifest.Files {
|
||||
if file.Path == path {
|
||||
sourceSHA = file.SHA256
|
||||
sourceSize = file.Size
|
||||
break
|
||||
}
|
||||
}
|
||||
output := state.CatalogOutputFile{
|
||||
Path: path,
|
||||
PipelineID: pipelineID,
|
||||
DestinationID: destinationID,
|
||||
Source: state.CatalogSourceIdentity{
|
||||
ID: req.SourceBundle.Manifest.ID,
|
||||
Digest: req.SourceBundle.Manifest.Digest,
|
||||
Created: req.SourceBundle.Manifest.Created,
|
||||
},
|
||||
Kind: kind,
|
||||
SHA256: sourceSHA,
|
||||
Size: sourceSize,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func writeCatalogState(t *testing.T, backend *fake.Backend, relative string, catalog state.CatalogState) {
|
||||
t.Helper()
|
||||
writeJSONState(t, backend, relative, catalog)
|
||||
}
|
||||
|
||||
func legacyStateDocument(schemaVersion int) map[string]int {
|
||||
return map[string]int{"schema_version": schemaVersion}
|
||||
}
|
||||
|
||||
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal state: %v", err)
|
||||
}
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||
}
|
||||
@@ -18,7 +18,11 @@ func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath
|
||||
if parseErr != nil {
|
||||
return state.DestinationStatus{StateErr: parseErr}, nil
|
||||
}
|
||||
return state.DestinationStatus{State: document.SingleOwner, SharedRoot: document.SharedRoot, HasContents: true}, nil
|
||||
return state.DestinationStatus{
|
||||
Catalog: document.Catalog,
|
||||
SupersededLegacy: document.SupersededLegacy,
|
||||
HasContents: true,
|
||||
}, nil
|
||||
}
|
||||
if !storage.IsNotFound(err) {
|
||||
return state.DestinationStatus{}, err
|
||||
|
||||
@@ -1,399 +0,0 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestBuildSharedRootTreatsAbsentOwnerAsPublishable(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionPublishNew {
|
||||
t.Fatalf("plan action = %s, want publish_new", plan.Action)
|
||||
}
|
||||
if plan.OwnerScope != state.CurrentOwnerScope("reports", "archive") {
|
||||
t.Fatalf("owner scope = %#v, want reports/archive", plan.OwnerScope)
|
||||
}
|
||||
if got, want := len(plan.OtherOwnerOutputs), 1; got != want {
|
||||
t.Fatalf("other owner output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(plan.OwnerOutputsToWrite), 1; got != want {
|
||||
t.Fatalf("owner output write count = %d, want %d", got, want)
|
||||
}
|
||||
if len(plan.OwnerOutputsToDelete) != 0 || len(plan.RetainedOwnerOutputs) != 0 {
|
||||
t.Fatalf("delete=%#v retained=%#v, want none", plan.OwnerOutputsToDelete, plan.RetainedOwnerOutputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.OwnerOutputsToDelete), "old.md"; got != want {
|
||||
t.Fatalf("owner outputs to delete = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.OtherOwnerOutputs), "other/report.md"; got != want {
|
||||
t.Fatalf("other owner outputs = %q, want %q", got, want)
|
||||
}
|
||||
if len(plan.RetainedOwnerOutputs) != 0 {
|
||||
t.Fatalf("retained owner outputs = %#v, want none", plan.RetainedOwnerOutputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.RetainedOwnerOutputs), "old.md"; got != want {
|
||||
t.Fatalf("retained owner outputs = %q, want %q", got, want)
|
||||
}
|
||||
if len(plan.OwnerOutputsToDelete) != 0 {
|
||||
t.Fatalf("owner outputs to delete = %#v, want none", plan.OwnerOutputsToDelete)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootRejectsOtherOwnerPathConflict(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
|
||||
sharedRoot.Outputs[0].Path = "report.md"
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
|
||||
t.Fatalf("Build() error = %v, want fail_conflict", err)
|
||||
}
|
||||
if plan.Action != ActionFailConflict {
|
||||
t.Fatalf("plan action = %s, want fail_conflict", plan.Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootRejectsUnmanagedPathCollision(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/report.md", "unmanaged")
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
|
||||
t.Fatalf("Build() error = %v, want fail_unmanaged", err)
|
||||
}
|
||||
if plan.Action != ActionFailUnmanaged {
|
||||
t.Fatalf("plan action = %s, want fail_unmanaged", plan.Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootPublishesOwnerAndPreservesOtherOwners(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := len(destinationState.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := destinationState.State.Mode, state.StateModeSharedRoot; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("other", "archive")); !ok {
|
||||
t.Fatal("other owner missing from shared-root state")
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports", "archive")); !ok {
|
||||
t.Fatal("current owner missing from shared-root state")
|
||||
}
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
existing := sharedRootStateWithOwners(t, sourceBundle.Manifest, true)
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", existing)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.md")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
if !destinationState.CreatedAt.Equal(existing.CreatedAt) {
|
||||
t.Fatalf("created_at = %s, want %s", destinationState.CreatedAt, existing.CreatedAt)
|
||||
}
|
||||
output, ok := findSharedRootOutputForTest(destinationState.Outputs, "report.md")
|
||||
if !ok {
|
||||
t.Fatal("report.md missing from shared-root outputs")
|
||||
}
|
||||
if !output.CreatedAt.Equal(existing.Outputs[1].CreatedAt) || !output.UpdatedAt.After(existing.Outputs[1].UpdatedAt) {
|
||||
t.Fatalf("report.md timestamps = created:%s updated:%s, want preserved created and newer updated", output.CreatedAt, output.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/old.md", "old")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md,old.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootForceReplaceDeletesOnlyBundlePath(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unmanaged.txt", "unmanaged")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace {
|
||||
t.Fatalf("plan action = %s, want force_replace", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unmanaged.txt")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := len(destinationState.Owners), 1; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func sharedRootRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
State: config.StatePolicy{Mode: config.StateModeSharedRoot},
|
||||
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
|
||||
Transfer: defaultTransfer(),
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string, sharedRoot state.SharedRootState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(sharedRoot, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared-root state: %v", err)
|
||||
}
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
path, err := storage.Join(relative, output.Path)
|
||||
if err != nil {
|
||||
t.Fatalf("join output path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, path, "old")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string) state.SharedRootState {
|
||||
t.Helper()
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
data, err := backend.ReadFile(context.Background(), statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
}
|
||||
destinationState, err := state.ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func findSharedRootOutputForTest(outputs []state.SharedRootOutputFile, path string) (state.SharedRootOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return state.SharedRootOutputFile{}, false
|
||||
}
|
||||
|
||||
func sharedRootStateWithOwners(t *testing.T, current bundle.Manifest, includeCurrent bool) state.SharedRootState {
|
||||
t.Helper()
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
otherManifest := testutil.ValidManifest(testutil.BundleOptions{
|
||||
ID: "other.source",
|
||||
Files: []testutil.SourceFile{{Path: "other/report.md", Data: "# Other\n"}},
|
||||
})
|
||||
sharedRoot := state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: state.CurrentOwnerScope("other", "archive"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: otherManifest},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "other/report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "other/report.md",
|
||||
SHA256: otherManifest.Files[0].SHA256,
|
||||
Size: otherManifest.Files[0].Size,
|
||||
Owner: state.CurrentOwnerScope("other", "archive"),
|
||||
SourceID: otherManifest.ID,
|
||||
SourceDigest: otherManifest.Digest,
|
||||
SourceCreated: otherManifest.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
if !includeCurrent {
|
||||
return sharedRoot
|
||||
}
|
||||
older := current
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), current.Files...)
|
||||
older.Files = append(older.Files, bundle.ManifestFile{
|
||||
Path: "old.md",
|
||||
SHA256: bundle.FileDigest([]byte("old\n")),
|
||||
Size: int64(len("old\n")),
|
||||
})
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
scope := state.CurrentOwnerScope("reports", "archive")
|
||||
sharedRoot.Owners = append(sharedRoot.Owners, state.OwnerRecord{
|
||||
Scope: scope,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: older},
|
||||
})
|
||||
for _, file := range older.Files {
|
||||
sharedRoot.Outputs = append(sharedRoot.Outputs, state.SharedRootOutputFile{
|
||||
Path: file.Path,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: file.Path,
|
||||
SHA256: file.SHA256,
|
||||
Size: file.Size,
|
||||
Owner: scope,
|
||||
SourceID: older.ID,
|
||||
SourceDigest: older.Digest,
|
||||
SourceCreated: older.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
})
|
||||
}
|
||||
return sharedRoot
|
||||
}
|
||||
|
||||
func sharedRootOutputPathList(outputs []state.SharedRootOutputFile) string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return strings.Join(paths, ",")
|
||||
}
|
||||
422
internal/state/catalog.go
Normal file
422
internal/state/catalog.go
Normal file
@@ -0,0 +1,422 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
type CatalogState struct {
|
||||
SchemaVersion int
|
||||
DistributorVersion string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
State StatePolicy
|
||||
Outputs []CatalogOutputFile
|
||||
}
|
||||
|
||||
type CatalogSourceIdentity struct {
|
||||
ID string
|
||||
Digest string
|
||||
Created time.Time
|
||||
}
|
||||
|
||||
type CatalogOutputFile struct {
|
||||
Path string
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
Source CatalogSourceIdentity
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type rawCatalogState struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
State *rawStatePolicy `json:"state"`
|
||||
Outputs []rawCatalogOutput `json:"outputs"`
|
||||
}
|
||||
|
||||
type rawCatalogOutput struct {
|
||||
Path *string `json:"path"`
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
Source *rawCatalogSourceIdentity `json:"source"`
|
||||
Kind *string `json:"kind"`
|
||||
SourcePath *string `json:"source_path"`
|
||||
Transform *string `json:"transform"`
|
||||
URL *string `json:"url"`
|
||||
SHA256 *string `json:"sha256"`
|
||||
Size *int64 `json:"size"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
}
|
||||
|
||||
type rawCatalogSourceIdentity struct {
|
||||
ID *string `json:"id"`
|
||||
Digest *string `json:"digest"`
|
||||
Created *string `json:"created"`
|
||||
}
|
||||
|
||||
func ParseCatalog(data []byte) (CatalogState, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
var raw rawCatalogState
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return CatalogState{}, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return CatalogState{}, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
state, err := parseCatalogRaw(raw)
|
||||
if err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
if err := ValidateCatalog(state); err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseCatalogRaw(raw rawCatalogState) (CatalogState, error) {
|
||||
if raw.SchemaVersion == nil {
|
||||
return CatalogState{}, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
state := CatalogState{
|
||||
SchemaVersion: *raw.SchemaVersion,
|
||||
DistributorVersion: raw.DistributorVersion,
|
||||
}
|
||||
if state.SchemaVersion != CatalogSchemaVersion {
|
||||
return CatalogState{}, fmt.Errorf("state schema_version must be %d", CatalogSchemaVersion)
|
||||
}
|
||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
||||
if err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
state.CreatedAt = createdAt
|
||||
state.UpdatedAt = updatedAt
|
||||
if raw.State == nil || raw.State.Mode == "" {
|
||||
return CatalogState{}, fmt.Errorf("state state.mode is required")
|
||||
}
|
||||
state.State.Mode = raw.State.Mode
|
||||
if raw.Outputs == nil {
|
||||
return CatalogState{}, fmt.Errorf("state outputs is required")
|
||||
}
|
||||
outputs, err := parseCatalogOutputs(raw.Outputs)
|
||||
if err != nil {
|
||||
return CatalogState{}, err
|
||||
}
|
||||
state.Outputs = outputs
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseCatalogOutputs(rawOutputs []rawCatalogOutput) ([]CatalogOutputFile, error) {
|
||||
outputs := make([]CatalogOutputFile, 0, len(rawOutputs))
|
||||
for index, raw := range rawOutputs {
|
||||
output, err := parseCatalogOutput(index, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func parseCatalogOutput(index int, raw rawCatalogOutput) (CatalogOutputFile, error) {
|
||||
if raw.Path == nil || *raw.Path == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||
}
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
source, err := parseCatalogSourceIdentity(index, raw.Source)
|
||||
if err != nil {
|
||||
return CatalogOutputFile{}, err
|
||||
}
|
||||
if raw.Kind == nil || *raw.Kind == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
||||
}
|
||||
switch *raw.Kind {
|
||||
case OutputKindSource:
|
||||
if raw.SourcePath != nil {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].source_path is only valid for generated output", index)
|
||||
}
|
||||
if raw.Transform != nil {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].transform is only valid for generated output", index)
|
||||
}
|
||||
case OutputKindGenerated:
|
||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required for generated output", index)
|
||||
}
|
||||
if raw.Transform == nil || *raw.Transform == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
}
|
||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
||||
}
|
||||
if raw.Size == nil {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||
}
|
||||
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
||||
if err != nil {
|
||||
return CatalogOutputFile{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return CatalogOutputFile{}, err
|
||||
}
|
||||
output := CatalogOutputFile{
|
||||
Path: *raw.Path,
|
||||
PipelineID: *raw.PipelineID,
|
||||
DestinationID: *raw.DestinationID,
|
||||
Source: source,
|
||||
Kind: *raw.Kind,
|
||||
SHA256: *raw.SHA256,
|
||||
Size: *raw.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if raw.SourcePath != nil {
|
||||
output.SourcePath = *raw.SourcePath
|
||||
}
|
||||
if raw.Transform != nil {
|
||||
output.Transform = *raw.Transform
|
||||
}
|
||||
if raw.URL != nil {
|
||||
if *raw.URL == "" {
|
||||
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].url must not be empty", index)
|
||||
}
|
||||
output.URL = *raw.URL
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func parseCatalogSourceIdentity(index int, raw *rawCatalogSourceIdentity) (CatalogSourceIdentity, error) {
|
||||
if raw == nil {
|
||||
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source is required", index)
|
||||
}
|
||||
if raw.ID == nil || *raw.ID == "" {
|
||||
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source.id is required", index)
|
||||
}
|
||||
if raw.Digest == nil || *raw.Digest == "" {
|
||||
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source.digest is required", index)
|
||||
}
|
||||
created, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source.created", index), raw.Created)
|
||||
if err != nil {
|
||||
return CatalogSourceIdentity{}, err
|
||||
}
|
||||
return CatalogSourceIdentity{
|
||||
ID: *raw.ID,
|
||||
Digest: *raw.Digest,
|
||||
Created: created,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s CatalogState) CreatedAtString() string {
|
||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s CatalogState) UpdatedAtString() string {
|
||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s CatalogSourceIdentity) CreatedString() string {
|
||||
return s.Created.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o CatalogOutputFile) CreatedAtString() string {
|
||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o CatalogOutputFile) UpdatedAtString() string {
|
||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s CatalogState) MarshalJSON() ([]byte, error) {
|
||||
type stateJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
State StatePolicy `json:"state"`
|
||||
Outputs []CatalogOutputFile `json:"outputs"`
|
||||
}
|
||||
return json.Marshal(stateJSON{
|
||||
SchemaVersion: s.SchemaVersion,
|
||||
DistributorVersion: s.DistributorVersion,
|
||||
CreatedAt: s.CreatedAtString(),
|
||||
UpdatedAt: s.UpdatedAtString(),
|
||||
State: s.State,
|
||||
Outputs: s.Outputs,
|
||||
})
|
||||
}
|
||||
|
||||
func (s CatalogSourceIdentity) MarshalJSON() ([]byte, error) {
|
||||
type sourceJSON struct {
|
||||
ID string `json:"id"`
|
||||
Digest string `json:"digest"`
|
||||
Created string `json:"created"`
|
||||
}
|
||||
return json.Marshal(sourceJSON{
|
||||
ID: s.ID,
|
||||
Digest: s.Digest,
|
||||
Created: s.CreatedString(),
|
||||
})
|
||||
}
|
||||
|
||||
func (o CatalogOutputFile) MarshalJSON() ([]byte, error) {
|
||||
type outputJSON struct {
|
||||
Path string `json:"path"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Source CatalogSourceIdentity `json:"source"`
|
||||
Kind string `json:"kind"`
|
||||
SourcePath string `json:"source_path,omitempty"`
|
||||
Transform string `json:"transform,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
return json.Marshal(outputJSON{
|
||||
Path: o.Path,
|
||||
PipelineID: o.PipelineID,
|
||||
DestinationID: o.DestinationID,
|
||||
Source: o.Source,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
CreatedAt: o.CreatedAtString(),
|
||||
UpdatedAt: o.UpdatedAtString(),
|
||||
})
|
||||
}
|
||||
|
||||
func ValidateCatalog(s CatalogState) error {
|
||||
if s.SchemaVersion != CatalogSchemaVersion {
|
||||
return fmt.Errorf("state schema_version must be %d", CatalogSchemaVersion)
|
||||
}
|
||||
if s.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state created_at is required")
|
||||
}
|
||||
if s.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state updated_at is required")
|
||||
}
|
||||
if s.State.Mode != StateModeCatalog {
|
||||
return fmt.Errorf("state state.mode must be %s", StateModeCatalog)
|
||||
}
|
||||
if s.Outputs == nil {
|
||||
return fmt.Errorf("state outputs is required")
|
||||
}
|
||||
seenPaths := make(map[string]struct{}, len(s.Outputs))
|
||||
for index, output := range s.Outputs {
|
||||
if err := validateCatalogOutput(index, output); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := seenPaths[output.Path]; exists {
|
||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seenPaths[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCatalogOutput(index int, output CatalogOutputFile) error {
|
||||
if err := storage.ValidatePath(output.Path); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
||||
}
|
||||
if output.PipelineID == "" {
|
||||
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if !config.IsSlugLikeID(output.PipelineID) {
|
||||
return fmt.Errorf("state outputs[%d].pipeline_id must be a slug-like identifier", index)
|
||||
}
|
||||
if output.DestinationID == "" {
|
||||
return fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
if !config.IsSlugLikeID(output.DestinationID) {
|
||||
return fmt.Errorf("state outputs[%d].destination_id must be a slug-like identifier", index)
|
||||
}
|
||||
if err := validateCatalogSourceIdentity(index, output.Source); err != nil {
|
||||
return err
|
||||
}
|
||||
switch output.Kind {
|
||||
case OutputKindSource:
|
||||
if output.SourcePath != "" {
|
||||
return fmt.Errorf("state outputs[%d].source_path is only valid for generated output", index)
|
||||
}
|
||||
if output.Transform != "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is only valid for generated output", index)
|
||||
}
|
||||
case OutputKindGenerated:
|
||||
if output.SourcePath == "" {
|
||||
return fmt.Errorf("state outputs[%d].source_path is required for generated output", index)
|
||||
}
|
||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
||||
}
|
||||
if output.Transform == "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
||||
}
|
||||
if output.URL != "" {
|
||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
||||
}
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||
}
|
||||
if output.Size < 0 {
|
||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||
}
|
||||
if output.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
||||
}
|
||||
if output.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCatalogSourceIdentity(index int, source CatalogSourceIdentity) error {
|
||||
if source.ID == "" {
|
||||
return fmt.Errorf("state outputs[%d].source.id is required", index)
|
||||
}
|
||||
if err := bundle.ValidateDigest(source.Digest); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source.digest: %w", index, err)
|
||||
}
|
||||
if source.Created.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].source.created is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
398
internal/state/catalog_test.go
Normal file
398
internal/state/catalog_test.go
Normal file
@@ -0,0 +1,398 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
)
|
||||
|
||||
func TestParseCatalogState(t *testing.T) {
|
||||
state, err := ParseCatalog([]byte(validCatalogStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCatalog() error = %v", err)
|
||||
}
|
||||
if got, want := state.SchemaVersion, CatalogSchemaVersion; got != want {
|
||||
t.Fatalf("schema version = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-06-19T12:00:00Z"; got != want {
|
||||
t.Fatalf("created_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.UpdatedAtString(), "2026-06-19T12:05:00Z"; got != want {
|
||||
t.Fatalf("updated_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeCatalog; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(state.Outputs), 2; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
sourceOutput := state.Outputs[0]
|
||||
if sourceOutput.SourcePath != "" || sourceOutput.Transform != "" {
|
||||
t.Fatalf("source output source_path=%q transform=%q, want omitted", sourceOutput.SourcePath, sourceOutput.Transform)
|
||||
}
|
||||
generatedOutput := state.Outputs[1]
|
||||
if generatedOutput.SourcePath != "report.md" || generatedOutput.Transform != "markdown_to_html" {
|
||||
t.Fatalf("generated output source_path=%q transform=%q", generatedOutput.SourcePath, generatedOutput.Transform)
|
||||
}
|
||||
if got, want := generatedOutput.Source.CreatedString(), "2026-05-30T11:10:00Z"; got != want {
|
||||
t.Fatalf("source created = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogMarshalIsDeterministic(t *testing.T) {
|
||||
data, err := json.Marshal(validCatalogState(t))
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
|
||||
if string(data) != want {
|
||||
t.Fatalf("json = %s, want %s", data, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
|
||||
catalog, err := ParseDocument([]byte(validCatalogStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(catalog) error = %v", err)
|
||||
}
|
||||
if catalog.Catalog == nil || catalog.SupersededLegacy != nil {
|
||||
t.Fatalf("catalog document = %#v", catalog)
|
||||
}
|
||||
|
||||
tests := map[string]string{
|
||||
"schema 1": schemaOnlyStateJSON(legacySchemaVersion),
|
||||
"schema 2": schemaOnlyStateJSON(legacyOwnerSchema),
|
||||
"schema 3": schemaOnlyStateJSON(legacyMultiOwnerSchema),
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
document, err := ParseDocument([]byte(body))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument() error = %v", err)
|
||||
}
|
||||
if document.SupersededLegacy == nil || document.Catalog != nil {
|
||||
t.Fatalf("document = %#v, want superseded legacy only", document)
|
||||
}
|
||||
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
|
||||
t.Fatalf("legacy schema version = %d, want 1 through 3", document.SupersededLegacy.SchemaVersion)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDocumentRejectsUnsupportedFutureSchema(t *testing.T) {
|
||||
body := strings.Replace(validCatalogStateJSON(t), `"schema_version": 4`, `"schema_version": 5`, 1)
|
||||
_, err := ParseDocument([]byte(body))
|
||||
assertStateErrorContains(t, err, "schema_version 5 is unsupported")
|
||||
}
|
||||
|
||||
func TestParseDocumentRejectsTrailingData(t *testing.T) {
|
||||
_, err := ParseDocument([]byte(validCatalogStateJSON(t) + "\n{}"))
|
||||
assertStateErrorContains(t, err, "trailing data")
|
||||
}
|
||||
|
||||
func TestParseCatalogRejectsMissingFields(t *testing.T) {
|
||||
tests := map[string]func(map[string]any){
|
||||
"schema_version": func(document map[string]any) {
|
||||
delete(document, "schema_version")
|
||||
},
|
||||
"created_at": func(document map[string]any) {
|
||||
delete(document, "created_at")
|
||||
},
|
||||
"updated_at": func(document map[string]any) {
|
||||
delete(document, "updated_at")
|
||||
},
|
||||
"state": func(document map[string]any) {
|
||||
delete(document, "state")
|
||||
},
|
||||
"outputs": func(document map[string]any) {
|
||||
delete(document, "outputs")
|
||||
},
|
||||
"path": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "path")
|
||||
},
|
||||
"pipeline_id": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "pipeline_id")
|
||||
},
|
||||
"destination_id": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "destination_id")
|
||||
},
|
||||
"source": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "source")
|
||||
},
|
||||
"source id": func(document map[string]any) {
|
||||
delete(firstCatalogSource(document), "id")
|
||||
},
|
||||
"source digest": func(document map[string]any) {
|
||||
delete(firstCatalogSource(document), "digest")
|
||||
},
|
||||
"source created": func(document map[string]any) {
|
||||
delete(firstCatalogSource(document), "created")
|
||||
},
|
||||
"kind": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "kind")
|
||||
},
|
||||
"sha256": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "sha256")
|
||||
},
|
||||
"size": func(document map[string]any) {
|
||||
delete(firstCatalogOutput(document), "size")
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
document := catalogStateObject(t)
|
||||
mutate(document)
|
||||
_, err := ParseCatalog(mustMarshalCatalogObject(t, document))
|
||||
assertStateErrorContains(t, err, "required")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCatalogRejectsMalformedTimestamps(t *testing.T) {
|
||||
tests := map[string]func(string) string{
|
||||
"created_at": func(body string) string {
|
||||
return strings.Replace(body, `"created_at": "2026-06-19T12:00:00Z"`, `"created_at": "June 19"`, 1)
|
||||
},
|
||||
"updated_at": func(body string) string {
|
||||
return strings.Replace(body, `"updated_at": "2026-06-19T12:05:00Z"`, `"updated_at": "June 19"`, 1)
|
||||
},
|
||||
"source created": func(body string) string {
|
||||
return strings.Replace(body, `"created": "2026-05-30T11:10:00Z"`, `"created": "May 30"`, 1)
|
||||
},
|
||||
"output created_at": func(body string) string {
|
||||
return strings.Replace(body, ` "created_at": "2026-06-19T12:00:00Z"`, ` "created_at": "June 19"`, 1)
|
||||
},
|
||||
"output updated_at": func(body string) string {
|
||||
return strings.Replace(body, ` "updated_at": "2026-06-19T12:05:00Z"`, ` "updated_at": "June 19"`, 1)
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := ParseCatalog([]byte(mutate(validCatalogStateJSON(t))))
|
||||
assertStateErrorContains(t, err, "RFC3339")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCatalogRejectsInvalidOutputRecords(t *testing.T) {
|
||||
tests := map[string]func(*CatalogState){
|
||||
"duplicate path": func(s *CatalogState) {
|
||||
s.Outputs[1].Path = s.Outputs[0].Path
|
||||
},
|
||||
"invalid output path": func(s *CatalogState) {
|
||||
s.Outputs[0].Path = "../report.md"
|
||||
},
|
||||
"invalid pipeline id": func(s *CatalogState) {
|
||||
s.Outputs[0].PipelineID = ".reports"
|
||||
},
|
||||
"invalid destination id": func(s *CatalogState) {
|
||||
s.Outputs[0].DestinationID = ".archive"
|
||||
},
|
||||
"missing source id": func(s *CatalogState) {
|
||||
s.Outputs[0].Source.ID = ""
|
||||
},
|
||||
"invalid source digest": func(s *CatalogState) {
|
||||
s.Outputs[0].Source.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
|
||||
},
|
||||
"missing source created": func(s *CatalogState) {
|
||||
s.Outputs[0].Source.Created = time.Time{}
|
||||
},
|
||||
"invalid kind": func(s *CatalogState) {
|
||||
s.Outputs[0].Kind = "document"
|
||||
},
|
||||
"generated missing source path": func(s *CatalogState) {
|
||||
s.Outputs[1].SourcePath = ""
|
||||
},
|
||||
"generated invalid source path": func(s *CatalogState) {
|
||||
s.Outputs[1].SourcePath = "../report.md"
|
||||
},
|
||||
"generated missing transform": func(s *CatalogState) {
|
||||
s.Outputs[1].Transform = ""
|
||||
},
|
||||
"source output source path": func(s *CatalogState) {
|
||||
s.Outputs[0].SourcePath = "report.md"
|
||||
},
|
||||
"source output transform": func(s *CatalogState) {
|
||||
s.Outputs[0].Transform = "markdown_to_html"
|
||||
},
|
||||
"invalid output digest": func(s *CatalogState) {
|
||||
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
|
||||
},
|
||||
"negative size": func(s *CatalogState) {
|
||||
s.Outputs[0].Size = -1
|
||||
},
|
||||
"invalid url": func(s *CatalogState) {
|
||||
s.Outputs[1].URL = "file:///tmp/report.html"
|
||||
},
|
||||
"missing created at": func(s *CatalogState) {
|
||||
s.Outputs[0].CreatedAt = time.Time{}
|
||||
},
|
||||
"missing updated at": func(s *CatalogState) {
|
||||
s.Outputs[0].UpdatedAt = time.Time{}
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
state := validCatalogState(t)
|
||||
mutate(&state)
|
||||
if err := ValidateCatalog(state); err == nil {
|
||||
t.Fatal("ValidateCatalog() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCatalogRejectsForbiddenFields(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"owners": `"owners": [],`,
|
||||
"sources": `"sources": [],`,
|
||||
"workflow": `"workflow": "additive",`,
|
||||
"source": `"source": {"manifest": {}},`,
|
||||
"manifest": `"manifest": {},`,
|
||||
"pipeline_id": `"pipeline_id": "reports",`,
|
||||
"destination_id": `"destination_id": "archive",`,
|
||||
"published_at": `"published_at": "2026-06-19T12:00:00Z",`,
|
||||
}
|
||||
for name, field := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := strings.Replace(validCatalogStateJSON(t), `"created_at":`, field+"\n "+`"created_at":`, 1)
|
||||
_, err := ParseCatalog([]byte(body))
|
||||
assertStateErrorContains(t, err, "unknown field")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCatalogRejectsForbiddenOutputFieldsForSourceOutput(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"source_path": `"source_path": "report.md",`,
|
||||
"transform": `"transform": "markdown_to_html",`,
|
||||
"empty url": `"url": "",`,
|
||||
}
|
||||
for name, field := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := strings.Replace(validCatalogStateJSON(t), `"kind": "source",`, `"kind": "source",`+"\n "+field, 1)
|
||||
_, err := ParseCatalog([]byte(body))
|
||||
if err == nil {
|
||||
t.Fatal("ParseCatalog() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func validCatalogStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(validCatalogState(t), "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal catalog state: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func catalogStateObject(t *testing.T) map[string]any {
|
||||
t.Helper()
|
||||
var document map[string]any
|
||||
if err := json.Unmarshal([]byte(validCatalogStateJSON(t)), &document); err != nil {
|
||||
t.Fatalf("unmarshal catalog state: %v", err)
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
func firstCatalogOutput(document map[string]any) map[string]any {
|
||||
outputs := document["outputs"].([]any)
|
||||
return outputs[0].(map[string]any)
|
||||
}
|
||||
|
||||
func firstCatalogSource(document map[string]any) map[string]any {
|
||||
return firstCatalogOutput(document)["source"].(map[string]any)
|
||||
}
|
||||
|
||||
func mustMarshalCatalogObject(t *testing.T, document map[string]any) []byte {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal catalog object: %v", err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func schemaOnlyStateJSON(schemaVersion int) string {
|
||||
data, err := json.Marshal(map[string]int{"schema_version": schemaVersion})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func assertStateErrorContains(t *testing.T, err error, substring string) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("error = nil, want substring %q", substring)
|
||||
}
|
||||
if !strings.Contains(err.Error(), substring) {
|
||||
t.Fatalf("error = %v, want substring %q", err, substring)
|
||||
}
|
||||
}
|
||||
|
||||
func validManifest(t *testing.T) bundle.Manifest {
|
||||
t.Helper()
|
||||
files := []bundle.ManifestFile{{
|
||||
Path: "report.md",
|
||||
SHA256: bundle.FileDigest([]byte("# Report\nSunny.\n")),
|
||||
Size: int64(len("# Report\nSunny.\n")),
|
||||
}}
|
||||
manifest := bundle.Manifest{
|
||||
SchemaVersion: bundle.SchemaVersion,
|
||||
ID: "weather.daily.brentwood.2026-05-30",
|
||||
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||
Files: files,
|
||||
}
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
}
|
||||
|
||||
func validCatalogState(t *testing.T) CatalogState {
|
||||
t.Helper()
|
||||
manifest := validManifest(t)
|
||||
createdAt := time.Date(2026, 6, 19, 12, 0, 0, 0, time.UTC)
|
||||
updatedAt := time.Date(2026, 6, 19, 12, 5, 0, 0, time.UTC)
|
||||
source := CatalogSourceIdentity{
|
||||
ID: manifest.ID,
|
||||
Digest: manifest.Digest,
|
||||
Created: manifest.Created,
|
||||
}
|
||||
return CatalogState{
|
||||
SchemaVersion: CatalogSchemaVersion,
|
||||
DistributorVersion: "dev",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
State: StatePolicy{Mode: StateModeCatalog},
|
||||
Outputs: []CatalogOutputFile{{
|
||||
Path: "report.md",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: OutputKindSource,
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "html",
|
||||
Source: source,
|
||||
Kind: OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
URL: "https://reports.example.com/report.html",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 128,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
)
|
||||
|
||||
type Outcome string
|
||||
|
||||
const (
|
||||
OutcomeDestinationAbsent Outcome = "destination_absent"
|
||||
OutcomeDestinationUnmanaged Outcome = "destination_unmanaged"
|
||||
OutcomeInvalidState Outcome = "invalid_destination_state"
|
||||
OutcomeIdentityMismatch Outcome = "destination_identity_mismatch"
|
||||
OutcomeSameSource Outcome = "same_source_manifest"
|
||||
OutcomeDestinationOlder Outcome = "destination_older"
|
||||
OutcomeDestinationNewer Outcome = "destination_newer"
|
||||
OutcomeSameCreatedConflict Outcome = "same_created_digest_conflict"
|
||||
OutcomeDifferentSourceConflict Outcome = "different_source_conflict"
|
||||
)
|
||||
|
||||
type DestinationStatus struct {
|
||||
State *DistributorState
|
||||
SharedRoot *SharedRootState
|
||||
StateErr error
|
||||
HasContents bool
|
||||
}
|
||||
|
||||
type Comparison struct {
|
||||
Outcome Outcome
|
||||
Reason string
|
||||
}
|
||||
|
||||
func CompareSharedRootOwner(source bundle.Manifest, scope OwnerScope, status DestinationStatus) Comparison {
|
||||
if status.StateErr != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error()}
|
||||
}
|
||||
if status.SharedRoot != nil {
|
||||
if err := ValidateSharedRoot(*status.SharedRoot); err != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
|
||||
}
|
||||
owner, ok := status.SharedRoot.Owner(scope)
|
||||
if !ok {
|
||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: fmt.Sprintf("destination owner %s/%s is absent", scope.PipelineID, scope.DestinationID)}
|
||||
}
|
||||
return compareManifests(source, owner.Source.Manifest)
|
||||
}
|
||||
if status.State != nil {
|
||||
destinationState := *status.State
|
||||
if err := Validate(destinationState); err != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
|
||||
}
|
||||
if destinationState.PipelineID != scope.PipelineID {
|
||||
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationState.PipelineID, scope.PipelineID)}
|
||||
}
|
||||
if destinationState.DestinationID != scope.DestinationID {
|
||||
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("destination id %q does not match %q", destinationState.DestinationID, scope.DestinationID)}
|
||||
}
|
||||
return compareManifests(source, destinationState.Source.Manifest)
|
||||
}
|
||||
if status.HasContents {
|
||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state"}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
||||
}
|
||||
|
||||
func compareManifests(source, destination bundle.Manifest) Comparison {
|
||||
if manifestsEqual(source, destination) {
|
||||
return Comparison{Outcome: OutcomeSameSource, Reason: "destination source manifest matches source"}
|
||||
}
|
||||
if destination.ID != source.ID {
|
||||
return Comparison{Outcome: OutcomeDifferentSourceConflict, Reason: "destination source id differs from source"}
|
||||
}
|
||||
if destination.Created.Before(source.Created) {
|
||||
return Comparison{Outcome: OutcomeDestinationOlder, Reason: "destination source is older than source"}
|
||||
}
|
||||
if destination.Created.After(source.Created) {
|
||||
return Comparison{Outcome: OutcomeDestinationNewer, Reason: "destination source is newer than source"}
|
||||
}
|
||||
if destination.Digest != source.Digest {
|
||||
return Comparison{Outcome: OutcomeSameCreatedConflict, Reason: "destination source has same id and created time but different digest"}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: "destination source differs from source without a supported comparison outcome"}
|
||||
}
|
||||
|
||||
func Compare(source bundle.Manifest, pipelineID, destinationID string, status DestinationStatus) Comparison {
|
||||
if status.StateErr != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error()}
|
||||
}
|
||||
if status.State == nil {
|
||||
if status.HasContents {
|
||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state"}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
||||
}
|
||||
|
||||
destinationState := *status.State
|
||||
if err := Validate(destinationState); err != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
|
||||
}
|
||||
if destinationState.PipelineID != pipelineID {
|
||||
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationState.PipelineID, pipelineID)}
|
||||
}
|
||||
if destinationState.DestinationID != destinationID {
|
||||
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("destination id %q does not match %q", destinationState.DestinationID, destinationID)}
|
||||
}
|
||||
|
||||
return compareManifests(source, destinationState.Source.Manifest)
|
||||
}
|
||||
|
||||
func manifestsEqual(a, b bundle.Manifest) bool {
|
||||
if a.SchemaVersion != b.SchemaVersion ||
|
||||
a.ID != b.ID ||
|
||||
a.Digest != b.Digest ||
|
||||
!a.Created.Equal(b.Created) ||
|
||||
len(a.Files) != len(b.Files) {
|
||||
return false
|
||||
}
|
||||
for index := range a.Files {
|
||||
if a.Files[index] != b.Files[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestCompareOutcomes(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
tests := []struct {
|
||||
name string
|
||||
status DestinationStatus
|
||||
want Outcome
|
||||
}{
|
||||
{
|
||||
name: "destination absent",
|
||||
status: DestinationStatus{},
|
||||
want: OutcomeDestinationAbsent,
|
||||
},
|
||||
{
|
||||
name: "destination unmanaged",
|
||||
status: DestinationStatus{HasContents: true},
|
||||
want: OutcomeDestinationUnmanaged,
|
||||
},
|
||||
{
|
||||
name: "invalid destination state",
|
||||
status: DestinationStatus{StateErr: errors.New("invalid json")},
|
||||
want: OutcomeInvalidState,
|
||||
},
|
||||
{
|
||||
name: "pipeline mismatch",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
|
||||
want: OutcomeIdentityMismatch,
|
||||
},
|
||||
{
|
||||
name: "destination mismatch",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
|
||||
want: OutcomeIdentityMismatch,
|
||||
},
|
||||
{
|
||||
name: "same source manifest",
|
||||
status: DestinationStatus{State: withState(t, source, nil)},
|
||||
want: OutcomeSameSource,
|
||||
},
|
||||
{
|
||||
name: "destination older",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.Created = source.Created.Add(-time.Hour)
|
||||
})},
|
||||
want: OutcomeDestinationOlder,
|
||||
},
|
||||
{
|
||||
name: "destination newer",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.Created = source.Created.Add(time.Hour)
|
||||
})},
|
||||
want: OutcomeDestinationNewer,
|
||||
},
|
||||
{
|
||||
name: "same created digest conflict",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
||||
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
|
||||
})},
|
||||
want: OutcomeSameCreatedConflict,
|
||||
},
|
||||
{
|
||||
name: "different source id",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Source.Manifest.ID = "other.source"
|
||||
})},
|
||||
want: OutcomeDifferentSourceConflict,
|
||||
},
|
||||
{
|
||||
name: "invalid state object",
|
||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||
s.Outputs[0].Kind = "other"
|
||||
})},
|
||||
want: OutcomeInvalidState,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := Compare(source, "reports", "archive", tt.status)
|
||||
if got.Outcome != tt.want {
|
||||
t.Fatalf("Compare() outcome = %s reason=%q, want %s", got.Outcome, got.Reason, tt.want)
|
||||
}
|
||||
if got.Reason == "" {
|
||||
t.Fatal("Compare() reason is empty")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func withState(t *testing.T, source bundle.Manifest, mutate func(*DistributorState)) *DistributorState {
|
||||
t.Helper()
|
||||
stateManifest := source
|
||||
stateManifest.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: stateManifest},
|
||||
Outputs: []OutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
if mutate != nil {
|
||||
mutate(&state)
|
||||
}
|
||||
return &state
|
||||
}
|
||||
@@ -1,371 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
SchemaVersion = 2
|
||||
SharedRootSchemaVersion = 3
|
||||
legacySchemaVersion = 1
|
||||
StateModeSingleOwner = config.StateModeSingleOwner
|
||||
StateModeSharedRoot = config.StateModeSharedRoot
|
||||
)
|
||||
|
||||
type DistributorState struct {
|
||||
SchemaVersion int
|
||||
DistributorVersion string
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
PublishedAt time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
State StatePolicy
|
||||
Reconciliation ReconciliationPolicy
|
||||
Source SourceState
|
||||
Links *LinkState
|
||||
Outputs []OutputFile
|
||||
}
|
||||
|
||||
type StatePolicy struct {
|
||||
Mode string
|
||||
}
|
||||
|
||||
type ReconciliationPolicy struct {
|
||||
Mode string
|
||||
}
|
||||
|
||||
type SourceState struct {
|
||||
Manifest bundle.Manifest
|
||||
}
|
||||
|
||||
type LinkState struct {
|
||||
PrimaryURL string
|
||||
}
|
||||
|
||||
type OutputFile struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type rawDistributorState struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version"`
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
PublishedAt *string `json:"published_at"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
State *rawStatePolicy `json:"state"`
|
||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
||||
Source *rawSourceState `json:"source"`
|
||||
Links *rawLinkState `json:"links"`
|
||||
Outputs []rawOutputFile `json:"outputs"`
|
||||
}
|
||||
|
||||
type rawStatePolicy struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
type rawReconciliationPolicy struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
type rawSourceState struct {
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
}
|
||||
|
||||
type rawLinkState struct {
|
||||
PrimaryURL string `json:"primary_url"`
|
||||
}
|
||||
|
||||
type rawOutputFile struct {
|
||||
Path *string `json:"path"`
|
||||
Kind *string `json:"kind"`
|
||||
SourcePath *string `json:"source_path"`
|
||||
Transform string `json:"transform"`
|
||||
URL string `json:"url"`
|
||||
SHA256 *string `json:"sha256"`
|
||||
Size *int64 `json:"size"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func Parse(data []byte) (DistributorState, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw rawDistributorState
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return DistributorState{}, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return DistributorState{}, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
state, err := parseRaw(raw)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
if err := Validate(state); err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
||||
var state DistributorState
|
||||
if raw.SchemaVersion == nil {
|
||||
return DistributorState{}, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
state.SchemaVersion = *raw.SchemaVersion
|
||||
if state.SchemaVersion != SchemaVersion && state.SchemaVersion != legacySchemaVersion {
|
||||
return DistributorState{}, fmt.Errorf("state schema_version must be %d or %d", legacySchemaVersion, SchemaVersion)
|
||||
}
|
||||
legacy := state.SchemaVersion == legacySchemaVersion
|
||||
state.DistributorVersion = raw.DistributorVersion
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return DistributorState{}, fmt.Errorf("state pipeline_id is required")
|
||||
}
|
||||
state.PipelineID = *raw.PipelineID
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return DistributorState{}, fmt.Errorf("state destination_id is required")
|
||||
}
|
||||
state.DestinationID = *raw.DestinationID
|
||||
if raw.PublishedAt == nil || *raw.PublishedAt == "" {
|
||||
return DistributorState{}, fmt.Errorf("state published_at is required")
|
||||
}
|
||||
publishedAt, err := time.Parse(time.RFC3339, *raw.PublishedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, fmt.Errorf("state published_at must be RFC3339: %w", err)
|
||||
}
|
||||
state.PublishedAt = publishedAt.UTC()
|
||||
if legacy {
|
||||
state.SchemaVersion = SchemaVersion
|
||||
state.CreatedAt = state.PublishedAt
|
||||
state.UpdatedAt = state.PublishedAt
|
||||
state.State.Mode = StateModeSingleOwner
|
||||
state.Reconciliation.Mode = config.ReconciliationModeReplace
|
||||
} else {
|
||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
state.CreatedAt = createdAt
|
||||
state.UpdatedAt = updatedAt
|
||||
if raw.State == nil || raw.State.Mode == "" {
|
||||
return DistributorState{}, fmt.Errorf("state state.mode is required")
|
||||
}
|
||||
state.State.Mode = raw.State.Mode
|
||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
||||
return DistributorState{}, fmt.Errorf("state reconciliation.mode is required")
|
||||
}
|
||||
state.Reconciliation.Mode = raw.Reconciliation.Mode
|
||||
}
|
||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
||||
return DistributorState{}, fmt.Errorf("state source.manifest is required")
|
||||
}
|
||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
||||
if err != nil {
|
||||
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
|
||||
}
|
||||
state.Source.Manifest = manifest
|
||||
if raw.Links != nil {
|
||||
state.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
||||
}
|
||||
if raw.Outputs == nil {
|
||||
return DistributorState{}, fmt.Errorf("state outputs is required")
|
||||
}
|
||||
outputs, err := parseOutputs(raw.Outputs, legacy, state.PublishedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
state.Outputs = outputs
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseRequiredTime(context string, raw *string) (time.Time, error) {
|
||||
if raw == nil || *raw == "" {
|
||||
return time.Time{}, fmt.Errorf("%s is required", context)
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, *raw)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", context, err)
|
||||
}
|
||||
return parsed.UTC(), nil
|
||||
}
|
||||
|
||||
func parseOutputs(rawOutputs []rawOutputFile, legacy bool, publishedAt time.Time) ([]OutputFile, error) {
|
||||
outputs := make([]OutputFile, 0, len(rawOutputs))
|
||||
seen := make(map[string]struct{}, len(rawOutputs))
|
||||
for index, raw := range rawOutputs {
|
||||
output, err := parseOutput(index, raw, legacy, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, exists := seen[output.Path]; exists {
|
||||
return nil, fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seen[output.Path] = struct{}{}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func parseOutput(index int, raw rawOutputFile, legacy bool, publishedAt time.Time) (OutputFile, error) {
|
||||
if raw.Path == nil || *raw.Path == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||
}
|
||||
if raw.Kind == nil || *raw.Kind == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
||||
}
|
||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
||||
}
|
||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
||||
}
|
||||
if raw.Size == nil {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||
}
|
||||
createdAt := publishedAt
|
||||
updatedAt := publishedAt
|
||||
if !legacy {
|
||||
var err error
|
||||
createdAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
||||
if err != nil {
|
||||
return OutputFile{}, err
|
||||
}
|
||||
updatedAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return OutputFile{}, err
|
||||
}
|
||||
}
|
||||
return OutputFile{
|
||||
Path: *raw.Path,
|
||||
Kind: *raw.Kind,
|
||||
SourcePath: *raw.SourcePath,
|
||||
Transform: raw.Transform,
|
||||
URL: raw.URL,
|
||||
SHA256: *raw.SHA256,
|
||||
Size: *raw.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s DistributorState) PublishedAtString() string {
|
||||
return s.PublishedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) CreatedAtString() string {
|
||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) UpdatedAtString() string {
|
||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o OutputFile) CreatedAtString() string {
|
||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o OutputFile) UpdatedAtString() string {
|
||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) MarshalJSON() ([]byte, error) {
|
||||
type sourceJSON struct {
|
||||
Manifest bundle.Manifest `json:"manifest"`
|
||||
}
|
||||
type stateJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
State StatePolicy `json:"state"`
|
||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
||||
Source sourceJSON `json:"source"`
|
||||
Links *LinkState `json:"links,omitempty"`
|
||||
Outputs []OutputFile `json:"outputs"`
|
||||
}
|
||||
return json.Marshal(stateJSON{
|
||||
SchemaVersion: s.SchemaVersion,
|
||||
DistributorVersion: s.DistributorVersion,
|
||||
PipelineID: s.PipelineID,
|
||||
DestinationID: s.DestinationID,
|
||||
PublishedAt: s.PublishedAtString(),
|
||||
CreatedAt: s.CreatedAtString(),
|
||||
UpdatedAt: s.UpdatedAtString(),
|
||||
State: s.State,
|
||||
Reconciliation: s.Reconciliation,
|
||||
Source: sourceJSON{Manifest: s.Source.Manifest},
|
||||
Links: s.Links,
|
||||
Outputs: s.Outputs,
|
||||
})
|
||||
}
|
||||
|
||||
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
||||
type policyJSON struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||
}
|
||||
|
||||
func (p ReconciliationPolicy) MarshalJSON() ([]byte, error) {
|
||||
type policyJSON struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||
}
|
||||
|
||||
func (l LinkState) MarshalJSON() ([]byte, error) {
|
||||
type linkJSON struct {
|
||||
PrimaryURL string `json:"primary_url,omitempty"`
|
||||
}
|
||||
return json.Marshal(linkJSON{PrimaryURL: l.PrimaryURL})
|
||||
}
|
||||
|
||||
func (o OutputFile) MarshalJSON() ([]byte, error) {
|
||||
type outputJSON struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
SourcePath string `json:"source_path"`
|
||||
Transform string `json:"transform,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
return json.Marshal(outputJSON{
|
||||
Path: o.Path,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
CreatedAt: o.CreatedAt.UTC().Format(time.RFC3339),
|
||||
UpdatedAt: o.UpdatedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
@@ -1,473 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestParseValidState(t *testing.T) {
|
||||
state, err := Parse([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if state.SchemaVersion != SchemaVersion {
|
||||
t.Fatalf("schema version = %d, want %d", state.SchemaVersion, SchemaVersion)
|
||||
}
|
||||
if state.PipelineID != "reports" || state.DestinationID != "archive" {
|
||||
t.Fatalf("identity = %q/%q", state.PipelineID, state.DestinationID)
|
||||
}
|
||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("CreatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("UpdatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(state.Outputs), 1; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := state.Outputs[0].CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("output CreatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseValidStateWithLinks(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"outputs": [`, `"links": {"primary_url": "https://reports.example.com/archive/report.md"},`+"\n "+`"outputs": [`, 1)
|
||||
body = strings.Replace(body, `"source_path": "report.md",`, `"source_path": "report.md",`+"\n "+`"url": "https://reports.example.com/archive/report.md",`, 1)
|
||||
|
||||
state, err := Parse([]byte(body))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if state.Links == nil || state.Links.PrimaryURL != "https://reports.example.com/archive/report.md" {
|
||||
t.Fatalf("links = %#v, want primary URL", state.Links)
|
||||
}
|
||||
if state.Outputs[0].URL != "https://reports.example.com/archive/report.md" {
|
||||
t.Fatalf("output URL = %q", state.Outputs[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingOutputs(t *testing.T) {
|
||||
state, err := Parse([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
state.Outputs = append(state.Outputs, OutputFile{
|
||||
Path: "summary.txt",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 10,
|
||||
CreatedAt: state.CreatedAt,
|
||||
UpdatedAt: state.UpdatedAt,
|
||||
})
|
||||
|
||||
next, changed := RemoveMissingOutputs(state, []string{"summary.txt"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingOutputs() changed = false, want true")
|
||||
}
|
||||
if got, want := ManagedOutputPaths(next), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
unchanged, changed := RemoveMissingOutputs(next, []string{"missing.txt"})
|
||||
if changed {
|
||||
t.Fatal("RemoveMissingOutputs() changed = true, want false")
|
||||
}
|
||||
if got, want := ManagedOutputPaths(unchanged), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("unchanged paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
|
||||
state, err := Parse([]byte(body))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsMissingFields(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"schema_version": `"schema_version"`,
|
||||
"pipeline_id": `"pipeline_id"`,
|
||||
"destination_id": `"destination_id"`,
|
||||
"published_at": `"published_at"`,
|
||||
"created_at": `"created_at"`,
|
||||
"updated_at": `"updated_at"`,
|
||||
"state": `"state"`,
|
||||
"reconciliation": `"reconciliation"`,
|
||||
"source": `"source"`,
|
||||
"outputs": `"outputs"`,
|
||||
}
|
||||
for name, field := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), field, `"missing_`+name+`"`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "required")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidSchemaVersion(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"schema_version": 2`, `"schema_version": 3`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "schema_version must be 1 or 2")
|
||||
}
|
||||
|
||||
func TestParseLegacyStateInfersSingleOwnerDefaults(t *testing.T) {
|
||||
state, err := Parse([]byte(legacyStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if got, want := state.SchemaVersion, SchemaVersion; got != want {
|
||||
t.Fatalf("schema version = %d, want normalized %d", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("created_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Outputs[0].UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("output updated_at = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidEmbeddedManifest(t *testing.T) {
|
||||
body := validStateWithManifestJSON(t, strings.Replace(manifestJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1))
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "source.manifest")
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidEmbeddedManifest(t *testing.T) {
|
||||
tests := map[string]func(bundle.Manifest) bundle.Manifest{
|
||||
"schema version": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.SchemaVersion = 2
|
||||
return manifest
|
||||
},
|
||||
"empty id": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.ID = ""
|
||||
return manifest
|
||||
},
|
||||
"bad digest": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
|
||||
return manifest
|
||||
},
|
||||
"zero created": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Created = time.Time{}
|
||||
return manifest
|
||||
},
|
||||
"empty files": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Files = nil
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
},
|
||||
"unsafe path": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Files[0].Path = "../report.md"
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
},
|
||||
"duplicate path": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Files[1].Path = manifest.Files[0].Path
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
},
|
||||
"negative size": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Files[0].Size = -1
|
||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||
return manifest
|
||||
},
|
||||
"digest mismatch": func(manifest bundle.Manifest) bundle.Manifest {
|
||||
manifest.Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
||||
return manifest
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
source := mutate(validManifest(t))
|
||||
state := *withState(t, validManifest(t), func(*DistributorState) {})
|
||||
state.Source.Manifest = source
|
||||
err := Validate(state)
|
||||
assertStateErrorContains(t, err, "source.manifest")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
tests := map[string]func(*DistributorState){
|
||||
"unsafe path": func(s *DistributorState) {
|
||||
s.Outputs[0].Path = "../report.md"
|
||||
},
|
||||
"invalid kind": func(s *DistributorState) {
|
||||
s.Outputs[0].Kind = "other"
|
||||
},
|
||||
"invalid source": func(s *DistributorState) {
|
||||
s.Outputs[0].SourcePath = "../report.md"
|
||||
},
|
||||
"generated missing": func(s *DistributorState) {
|
||||
s.Outputs[0].Kind = OutputKindGenerated
|
||||
},
|
||||
"invalid digest": func(s *DistributorState) {
|
||||
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
|
||||
},
|
||||
"negative size": func(s *DistributorState) {
|
||||
s.Outputs[0].Size = -1
|
||||
},
|
||||
"missing output created_at": func(s *DistributorState) {
|
||||
s.Outputs[0].CreatedAt = time.Time{}
|
||||
},
|
||||
"missing output updated_at": func(s *DistributorState) {
|
||||
s.Outputs[0].UpdatedAt = time.Time{}
|
||||
},
|
||||
"invalid output url": func(s *DistributorState) {
|
||||
s.Outputs[0].URL = "file:///tmp/report.md"
|
||||
},
|
||||
"invalid primary url": func(s *DistributorState) {
|
||||
s.Links = &LinkState{PrimaryURL: "file:///tmp/report.md"}
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
state := *withState(t, source, mutate)
|
||||
err := Validate(state)
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReportsURLFieldContext(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
state := *withState(t, source, func(s *DistributorState) {
|
||||
s.Links = &LinkState{PrimaryURL: "https://reports.example.com/archive#top"}
|
||||
})
|
||||
err := Validate(state)
|
||||
assertStateErrorContains(t, err, "state links.primary_url")
|
||||
assertStateErrorContains(t, err, "must not include a fragment")
|
||||
|
||||
state = *withState(t, source, func(s *DistributorState) {
|
||||
s.Outputs[0].URL = "https://reports.example.com/archive?preview=1"
|
||||
})
|
||||
err = Validate(state)
|
||||
assertStateErrorContains(t, err, "state outputs[0].url")
|
||||
assertStateErrorContains(t, err, "must not include a query string")
|
||||
}
|
||||
|
||||
func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "May 30"`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "published_at must be RFC3339")
|
||||
}
|
||||
|
||||
func TestParseRejectsMalformedCreatedTimestamp(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"created_at": "2026-05-30T11:12:00Z"`, `"created_at": "May 30"`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "created_at must be RFC3339")
|
||||
}
|
||||
|
||||
func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
publishedAt := time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Outputs: []OutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"published_at":"2026-05-30T11:12:00Z"`) {
|
||||
t.Fatalf("json = %s, want UTC RFC3339 published_at", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
||||
Outputs: []OutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
URL: "https://reports.example.com/archive/report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"links":{"primary_url":"https://reports.example.com/archive/report.md"}`) {
|
||||
t.Fatalf("json = %s, want links primary URL", data)
|
||||
}
|
||||
if !strings.Contains(string(data), `"url":"https://reports.example.com/archive/report.md"`) {
|
||||
t.Fatalf("json = %s, want output URL", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputHelpers(t *testing.T) {
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
updatedAt := createdAt.Add(time.Hour)
|
||||
retained := []OutputFile{{
|
||||
Path: "old.md",
|
||||
Kind: OutputKindSource,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}}
|
||||
projected := ProjectOutputs([]OutputProjection{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
}, {
|
||||
Path: "new.md",
|
||||
Kind: OutputKindSource,
|
||||
}}, retained, updatedAt)
|
||||
if got, ok := FindOutputByPath(projected, "report.md"); !ok || !got.CreatedAt.Equal(createdAt) || !got.UpdatedAt.Equal(updatedAt) {
|
||||
t.Fatalf("projected report.md = %#v, want preserved created_at and updated updated_at", got)
|
||||
}
|
||||
merged, err := MergeOutputFiles(retained, projected)
|
||||
if err != nil {
|
||||
t.Fatalf("MergeOutputFiles() error = %v", err)
|
||||
}
|
||||
if got, want := len(merged), 3; got != want {
|
||||
t.Fatalf("merged count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := ManagedOutputPaths(DistributorState{Outputs: merged}), []string{"old.md", "report.md", "new.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("managed paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func validStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
return validStateWithManifestJSON(t, manifestJSON(t))
|
||||
}
|
||||
|
||||
func validStateWithManifestJSON(t *testing.T, manifest string) string {
|
||||
t.Helper()
|
||||
return `{
|
||||
"schema_version": 2,
|
||||
"distributor_version": "dev",
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"published_at": "2026-05-30T11:12:00Z",
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z",
|
||||
"state": {"mode": "single_owner"},
|
||||
"reconciliation": {"mode": "replace"},
|
||||
"source": {
|
||||
"manifest": ` + manifest + `
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"path": "report.md",
|
||||
"kind": "source",
|
||||
"source_path": "report.md",
|
||||
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
||||
"size": 16,
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z"
|
||||
}
|
||||
]
|
||||
}`
|
||||
}
|
||||
|
||||
func legacyStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
return strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(validStateJSON(t),
|
||||
`"schema_version": 2`, `"schema_version": 1`, 1),
|
||||
` "created_at": "2026-05-30T11:12:00Z",
|
||||
`, "", 1),
|
||||
` "updated_at": "2026-05-30T11:12:00Z",
|
||||
`, "", 1),
|
||||
` "state": {"mode": "single_owner"},
|
||||
`, "", 1),
|
||||
` "reconciliation": {"mode": "replace"},
|
||||
`, "", 1),
|
||||
`,
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z"`, "", 1)
|
||||
}
|
||||
|
||||
func manifestJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("../bundle/testdata/valid_bundle/manifest.json")
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest fixture: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func validManifest(t *testing.T) bundle.Manifest {
|
||||
t.Helper()
|
||||
manifest, err := bundle.ParseManifest([]byte(manifestJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseManifest() error = %v", err)
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
|
||||
func assertStateErrorContains(t *testing.T, err error, want string) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("error = nil, want substring %q", want)
|
||||
}
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error = %q, want substring %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
107
internal/state/document.go
Normal file
107
internal/state/document.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
legacyOwnerSchema = 2
|
||||
legacyMultiOwnerSchema = 3
|
||||
CatalogSchemaVersion = 4
|
||||
legacySchemaVersion = 1
|
||||
StateModeCatalog = "catalog"
|
||||
OutputKindSource = "source"
|
||||
OutputKindGenerated = "generated"
|
||||
)
|
||||
|
||||
type StateDocument struct {
|
||||
Catalog *CatalogState
|
||||
SupersededLegacy *SupersededLegacyState
|
||||
}
|
||||
|
||||
type DestinationStatus struct {
|
||||
Catalog *CatalogState
|
||||
SupersededLegacy *SupersededLegacyState
|
||||
StateErr error
|
||||
HasContents bool
|
||||
}
|
||||
|
||||
type SupersededLegacyState struct {
|
||||
SchemaVersion int
|
||||
}
|
||||
|
||||
type StatePolicy struct {
|
||||
Mode string
|
||||
}
|
||||
|
||||
type OwnerScope struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
}
|
||||
|
||||
type rawStatePolicy struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
func ParseDocument(data []byte) (StateDocument, error) {
|
||||
schemaVersion, err := parseSchemaVersion(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
switch schemaVersion {
|
||||
case legacySchemaVersion, legacyOwnerSchema, legacyMultiOwnerSchema:
|
||||
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
|
||||
case CatalogSchemaVersion:
|
||||
catalog, err := ParseCatalog(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{Catalog: &catalog}, nil
|
||||
default:
|
||||
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func parseSchemaVersion(data []byte) (int, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
}
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return 0, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return 0, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
if raw.SchemaVersion == nil {
|
||||
return 0, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
return *raw.SchemaVersion, nil
|
||||
}
|
||||
|
||||
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
||||
type policyJSON struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||
}
|
||||
|
||||
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
||||
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
||||
}
|
||||
|
||||
func parseRequiredTime(field string, raw *string) (time.Time, error) {
|
||||
if raw == nil || *raw == "" {
|
||||
return time.Time{}, fmt.Errorf("%s is required", field)
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, *raw)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", field, err)
|
||||
}
|
||||
return parsed.UTC(), nil
|
||||
}
|
||||
@@ -1,138 +1,25 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
)
|
||||
|
||||
type OutputProjection struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
func FindOutputByPath(outputs []OutputFile, path string) (OutputFile, bool) {
|
||||
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return OutputFile{}, false
|
||||
return CatalogOutputFile{}, false
|
||||
}
|
||||
|
||||
func MergeOutputFiles(retained, planned []OutputFile) ([]OutputFile, error) {
|
||||
outputs := make([]OutputFile, 0, len(retained)+len(planned))
|
||||
indexByPath := make(map[string]int, len(retained)+len(planned))
|
||||
for _, output := range retained {
|
||||
if _, exists := indexByPath[output.Path]; exists {
|
||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
seenPlanned := make(map[string]struct{}, len(planned))
|
||||
for _, output := range planned {
|
||||
if _, exists := seenPlanned[output.Path]; exists {
|
||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
seenPlanned[output.Path] = struct{}{}
|
||||
if index, exists := indexByPath[output.Path]; exists {
|
||||
outputs[index] = output
|
||||
continue
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func ManagedOutputPaths(s DistributorState) []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func RemoveMissingOutputs(s DistributorState, missingPaths []string) (DistributorState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]OutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func ProjectOutputs(outputs []OutputProjection, existing []OutputFile, now time.Time) []OutputFile {
|
||||
now = now.UTC()
|
||||
files := make([]OutputFile, 0, len(outputs))
|
||||
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
|
||||
selected := make([]CatalogOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
createdAt := now
|
||||
if existingOutput, ok := FindOutputByPath(existing, output.Path); ok {
|
||||
createdAt = existingOutput.CreatedAt
|
||||
if output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
|
||||
selected = append(selected, output)
|
||||
}
|
||||
files = append(files, OutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
}
|
||||
return files
|
||||
return selected
|
||||
}
|
||||
|
||||
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
||||
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
||||
}
|
||||
|
||||
func (s SharedRootState) Owner(scope OwnerScope) (OwnerRecord, bool) {
|
||||
for _, owner := range s.Owners {
|
||||
if owner.Scope == scope {
|
||||
return owner, true
|
||||
}
|
||||
}
|
||||
return OwnerRecord{}, false
|
||||
}
|
||||
|
||||
func (s SharedRootState) SourceManifest(scope OwnerScope) (bundle.Manifest, bool) {
|
||||
owner, ok := s.Owner(scope)
|
||||
if !ok {
|
||||
return bundle.Manifest{}, false
|
||||
}
|
||||
return owner.Source.Manifest, true
|
||||
}
|
||||
|
||||
func (s SharedRootState) ManagedOutputPaths(scope OwnerScope) []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner == scope {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func (s SharedRootState) AllManagedOutputPaths() []string {
|
||||
func CatalogManagedOutputPaths(s CatalogState) []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
paths = append(paths, output.Path)
|
||||
@@ -140,16 +27,16 @@ func (s SharedRootState) AllManagedOutputPaths() []string {
|
||||
return paths
|
||||
}
|
||||
|
||||
func RemoveMissingSharedRootOwnerOutputs(s SharedRootState, scope OwnerScope, missingPaths []string) (SharedRootState, bool) {
|
||||
func RemoveMissingCatalogOwnerOutputs(s CatalogState, scope OwnerScope, missingPaths []string) (CatalogState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
||||
next.Outputs = make([]CatalogOutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner == scope {
|
||||
if output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
@@ -160,13 +47,13 @@ func RemoveMissingSharedRootOwnerOutputs(s SharedRootState, scope OwnerScope, mi
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (SharedRootState, bool) {
|
||||
func RemoveMissingCatalogOutputs(s CatalogState, missingPaths []string) (CatalogState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
||||
next.Outputs = make([]CatalogOutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
@@ -178,149 +65,6 @@ func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (S
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func (s SharedRootState) OutputOwner(path string) (OwnerScope, bool) {
|
||||
for _, output := range s.Outputs {
|
||||
if output.Path == path {
|
||||
return output.Owner, true
|
||||
}
|
||||
}
|
||||
return OwnerScope{}, false
|
||||
}
|
||||
|
||||
func (s SharedRootState) PathOwnershipConflict(scope OwnerScope, paths []string) (PathOwnershipConflict, bool) {
|
||||
for _, path := range paths {
|
||||
owner, exists := s.OutputOwner(path)
|
||||
if exists && owner != scope {
|
||||
return PathOwnershipConflict{Path: path, Owner: owner}, true
|
||||
}
|
||||
}
|
||||
return PathOwnershipConflict{}, false
|
||||
}
|
||||
|
||||
func ProjectSharedRootOutputs(outputs []OutputProjection, existing []SharedRootOutputFile, scope OwnerScope, source bundle.Manifest, now time.Time) []SharedRootOutputFile {
|
||||
now = now.UTC()
|
||||
files := make([]SharedRootOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
createdAt := now
|
||||
if existingOutput, ok := findSharedRootOutput(existing, output.Path); ok && existingOutput.Owner == scope {
|
||||
createdAt = existingOutput.CreatedAt
|
||||
}
|
||||
files = append(files, SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: scope,
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func ReplaceOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
}
|
||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
next := s
|
||||
next.Owners = upsertOwner(s.Owners, owner)
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner != scope {
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
}
|
||||
next.Outputs = append(next.Outputs, planned...)
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func MergeOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
}
|
||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
next := s
|
||||
next.Owners = upsertOwner(s.Owners, owner)
|
||||
outputs := make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
||||
indexByPath := make(map[string]int, len(s.Outputs)+len(planned))
|
||||
for _, output := range s.Outputs {
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
for _, output := range planned {
|
||||
if index, exists := indexByPath[output.Path]; exists {
|
||||
outputs[index] = output
|
||||
continue
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
next.Outputs = outputs
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func findSharedRootOutput(outputs []SharedRootOutputFile, path string) (SharedRootOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return SharedRootOutputFile{}, false
|
||||
}
|
||||
|
||||
func sharedRootOutputPaths(outputs []SharedRootOutputFile) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func rejectDuplicateSharedRootOutputs(outputs []SharedRootOutputFile) error {
|
||||
seen := make(map[string]struct{}, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if _, exists := seen[output.Path]; exists {
|
||||
return fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
seen[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePlannedSharedRootOutputs(scope OwnerScope, outputs []SharedRootOutputFile) error {
|
||||
if err := rejectDuplicateSharedRootOutputs(outputs); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, output := range outputs {
|
||||
if output.Owner != scope {
|
||||
return fmt.Errorf("state output path %q is owned by %s/%s, not %s/%s", output.Path, output.Owner.PipelineID, output.Owner.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertOwner(owners []OwnerRecord, owner OwnerRecord) []OwnerRecord {
|
||||
next := append([]OwnerRecord(nil), owners...)
|
||||
for index, existing := range next {
|
||||
if existing.Scope == owner.Scope {
|
||||
next[index] = owner
|
||||
return next
|
||||
}
|
||||
}
|
||||
return append(next, owner)
|
||||
}
|
||||
|
||||
func pathSet(paths []string) map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(paths))
|
||||
for _, path := range paths {
|
||||
|
||||
@@ -22,24 +22,13 @@ type PrunePlan struct {
|
||||
Preserved []PruneCandidate
|
||||
}
|
||||
|
||||
func SingleOwnerPruneCandidates(s DistributorState) []PruneCandidate {
|
||||
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
|
||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
candidates = append(candidates, PruneCandidate{
|
||||
Path: output.Path,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandidate {
|
||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner != scope {
|
||||
if output.PipelineID != scope.PipelineID || output.DestinationID != scope.DestinationID {
|
||||
continue
|
||||
}
|
||||
owner := output.Owner
|
||||
owner := scope
|
||||
candidates = append(candidates, PruneCandidate{
|
||||
Path: output.Path,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
|
||||
@@ -74,19 +74,6 @@ func TestPlanPruneDeterministicTieBreaking(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootPruneCandidatesPreserveOtherOwners(t *testing.T) {
|
||||
sharedRoot := validSharedRootState(t)
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
candidates := SharedRootPruneCandidates(sharedRoot, scope)
|
||||
|
||||
if got, want := pruneCandidatePaths(candidates), "report.md"; got != want {
|
||||
t.Fatalf("candidates = %q, want %q", got, want)
|
||||
}
|
||||
if candidates[0].Owner == nil || *candidates[0].Owner != scope {
|
||||
t.Fatalf("candidate owner = %#v, want current owner", candidates[0].Owner)
|
||||
}
|
||||
}
|
||||
|
||||
func pruneCandidatePaths(candidates []PruneCandidate) string {
|
||||
paths := make([]string, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
|
||||
@@ -1,519 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
type StateDocument struct {
|
||||
SingleOwner *DistributorState
|
||||
SharedRoot *SharedRootState
|
||||
}
|
||||
|
||||
type SharedRootState struct {
|
||||
SchemaVersion int
|
||||
DistributorVersion string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
State StatePolicy
|
||||
Owners []OwnerRecord
|
||||
Outputs []SharedRootOutputFile
|
||||
}
|
||||
|
||||
type OwnerScope struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
}
|
||||
|
||||
type OwnerRecord struct {
|
||||
Scope OwnerScope
|
||||
Reconciliation ReconciliationPolicy
|
||||
Source SourceState
|
||||
Links *LinkState
|
||||
}
|
||||
|
||||
type SharedRootOutputFile struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
Owner OwnerScope
|
||||
SourceID string
|
||||
SourceDigest string
|
||||
SourceCreated time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type PathOwnershipConflict struct {
|
||||
Path string
|
||||
Owner OwnerScope
|
||||
}
|
||||
|
||||
type rawSharedRootState struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
State *rawStatePolicy `json:"state"`
|
||||
Owners []rawOwnerRecord `json:"owners"`
|
||||
Outputs []rawSharedRootOutput `json:"outputs"`
|
||||
}
|
||||
|
||||
type rawOwnerRecord struct {
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
||||
Source *rawSourceState `json:"source"`
|
||||
Links *rawLinkState `json:"links"`
|
||||
}
|
||||
|
||||
type rawSharedRootOutput struct {
|
||||
Path *string `json:"path"`
|
||||
Kind *string `json:"kind"`
|
||||
SourcePath *string `json:"source_path"`
|
||||
Transform string `json:"transform"`
|
||||
URL string `json:"url"`
|
||||
SHA256 *string `json:"sha256"`
|
||||
Size *int64 `json:"size"`
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
SourceID *string `json:"source_id"`
|
||||
SourceDigest *string `json:"source_digest"`
|
||||
SourceCreated *string `json:"source_created"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func ParseDocument(data []byte) (StateDocument, error) {
|
||||
schemaVersion, err := parseSchemaVersion(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
if schemaVersion == SharedRootSchemaVersion {
|
||||
sharedRoot, err := ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{SharedRoot: &sharedRoot}, nil
|
||||
}
|
||||
singleOwner, err := Parse(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{SingleOwner: &singleOwner}, nil
|
||||
}
|
||||
|
||||
func parseSchemaVersion(data []byte) (int, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
}
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return 0, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
if raw.SchemaVersion == nil {
|
||||
return 0, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
return *raw.SchemaVersion, nil
|
||||
}
|
||||
|
||||
func ParseSharedRoot(data []byte) (SharedRootState, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw rawSharedRootState
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return SharedRootState{}, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return SharedRootState{}, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
state, err := parseSharedRootRaw(raw)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
if err := ValidateSharedRoot(state); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseSharedRootRaw(raw rawSharedRootState) (SharedRootState, error) {
|
||||
if raw.SchemaVersion == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
state := SharedRootState{SchemaVersion: *raw.SchemaVersion}
|
||||
if state.SchemaVersion != SharedRootSchemaVersion {
|
||||
return SharedRootState{}, fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
||||
}
|
||||
state.DistributorVersion = raw.DistributorVersion
|
||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.CreatedAt = createdAt
|
||||
state.UpdatedAt = updatedAt
|
||||
if raw.State == nil || raw.State.Mode == "" {
|
||||
return SharedRootState{}, fmt.Errorf("state state.mode is required")
|
||||
}
|
||||
state.State.Mode = raw.State.Mode
|
||||
if raw.Owners == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state owners is required")
|
||||
}
|
||||
owners, err := parseOwnerRecords(raw.Owners)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.Owners = owners
|
||||
if raw.Outputs == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state outputs is required")
|
||||
}
|
||||
outputs, err := parseSharedRootOutputs(raw.Outputs)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.Outputs = outputs
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseOwnerRecords(rawOwners []rawOwnerRecord) ([]OwnerRecord, error) {
|
||||
owners := make([]OwnerRecord, 0, len(rawOwners))
|
||||
for index, raw := range rawOwners {
|
||||
owner, err := parseOwnerRecord(index, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owners = append(owners, owner)
|
||||
}
|
||||
return owners, nil
|
||||
}
|
||||
|
||||
func parseOwnerRecord(index int, raw rawOwnerRecord) (OwnerRecord, error) {
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
||||
}
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].destination_id is required", index)
|
||||
}
|
||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].reconciliation.mode is required", index)
|
||||
}
|
||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest is required", index)
|
||||
}
|
||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
||||
if err != nil {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
||||
}
|
||||
owner := OwnerRecord{
|
||||
Scope: OwnerScope{
|
||||
PipelineID: *raw.PipelineID,
|
||||
DestinationID: *raw.DestinationID,
|
||||
},
|
||||
Reconciliation: ReconciliationPolicy{Mode: raw.Reconciliation.Mode},
|
||||
Source: SourceState{Manifest: manifest},
|
||||
}
|
||||
if raw.Links != nil {
|
||||
owner.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
||||
}
|
||||
return owner, nil
|
||||
}
|
||||
|
||||
func parseSharedRootOutputs(rawOutputs []rawSharedRootOutput) ([]SharedRootOutputFile, error) {
|
||||
outputs := make([]SharedRootOutputFile, 0, len(rawOutputs))
|
||||
for index, raw := range rawOutputs {
|
||||
output, err := parseSharedRootOutput(index, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func parseSharedRootOutput(index int, raw rawSharedRootOutput) (SharedRootOutputFile, error) {
|
||||
if raw.Path == nil || *raw.Path == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||
}
|
||||
if raw.Kind == nil || *raw.Kind == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
||||
}
|
||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
||||
}
|
||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
||||
}
|
||||
if raw.Size == nil {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||
}
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
if raw.SourceID == nil || *raw.SourceID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_id is required", index)
|
||||
}
|
||||
if raw.SourceDigest == nil || *raw.SourceDigest == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_digest is required", index)
|
||||
}
|
||||
sourceCreated, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source_created", index), raw.SourceCreated)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
return SharedRootOutputFile{
|
||||
Path: *raw.Path,
|
||||
Kind: *raw.Kind,
|
||||
SourcePath: *raw.SourcePath,
|
||||
Transform: raw.Transform,
|
||||
URL: raw.URL,
|
||||
SHA256: *raw.SHA256,
|
||||
Size: *raw.Size,
|
||||
Owner: OwnerScope{
|
||||
PipelineID: *raw.PipelineID,
|
||||
DestinationID: *raw.DestinationID,
|
||||
},
|
||||
SourceID: *raw.SourceID,
|
||||
SourceDigest: *raw.SourceDigest,
|
||||
SourceCreated: sourceCreated,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s SharedRootState) CreatedAtString() string {
|
||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s SharedRootState) UpdatedAtString() string {
|
||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) SourceCreatedString() string {
|
||||
return o.SourceCreated.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) CreatedAtString() string {
|
||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) UpdatedAtString() string {
|
||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s SharedRootState) MarshalJSON() ([]byte, error) {
|
||||
type stateJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
State StatePolicy `json:"state"`
|
||||
Owners []OwnerRecord `json:"owners"`
|
||||
Outputs []SharedRootOutputFile `json:"outputs"`
|
||||
}
|
||||
return json.Marshal(stateJSON{
|
||||
SchemaVersion: s.SchemaVersion,
|
||||
DistributorVersion: s.DistributorVersion,
|
||||
CreatedAt: s.CreatedAtString(),
|
||||
UpdatedAt: s.UpdatedAtString(),
|
||||
State: s.State,
|
||||
Owners: s.Owners,
|
||||
Outputs: s.Outputs,
|
||||
})
|
||||
}
|
||||
|
||||
func (o OwnerRecord) MarshalJSON() ([]byte, error) {
|
||||
type sourceJSON struct {
|
||||
Manifest bundle.Manifest `json:"manifest"`
|
||||
}
|
||||
type ownerJSON struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
||||
Source sourceJSON `json:"source"`
|
||||
Links *LinkState `json:"links,omitempty"`
|
||||
}
|
||||
return json.Marshal(ownerJSON{
|
||||
PipelineID: o.Scope.PipelineID,
|
||||
DestinationID: o.Scope.DestinationID,
|
||||
Reconciliation: o.Reconciliation,
|
||||
Source: sourceJSON{Manifest: o.Source.Manifest},
|
||||
Links: o.Links,
|
||||
})
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) MarshalJSON() ([]byte, error) {
|
||||
type outputJSON struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
SourcePath string `json:"source_path"`
|
||||
Transform string `json:"transform,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
SourceDigest string `json:"source_digest"`
|
||||
SourceCreated string `json:"source_created"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
return json.Marshal(outputJSON{
|
||||
Path: o.Path,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
PipelineID: o.Owner.PipelineID,
|
||||
DestinationID: o.Owner.DestinationID,
|
||||
SourceID: o.SourceID,
|
||||
SourceDigest: o.SourceDigest,
|
||||
SourceCreated: o.SourceCreatedString(),
|
||||
CreatedAt: o.CreatedAtString(),
|
||||
UpdatedAt: o.UpdatedAtString(),
|
||||
})
|
||||
}
|
||||
|
||||
func ValidateSharedRoot(s SharedRootState) error {
|
||||
if s.SchemaVersion != SharedRootSchemaVersion {
|
||||
return fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
||||
}
|
||||
if s.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state created_at is required")
|
||||
}
|
||||
if s.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state updated_at is required")
|
||||
}
|
||||
if s.State.Mode != StateModeSharedRoot {
|
||||
return fmt.Errorf("state state.mode must be %s", StateModeSharedRoot)
|
||||
}
|
||||
if s.Owners == nil {
|
||||
return fmt.Errorf("state owners is required")
|
||||
}
|
||||
owners := make(map[OwnerScope]OwnerRecord, len(s.Owners))
|
||||
for index, owner := range s.Owners {
|
||||
if err := validateOwnerRecord(index, owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := owners[owner.Scope]; exists {
|
||||
return fmt.Errorf("state owners[%d] duplicates owner %s/%s", index, owner.Scope.PipelineID, owner.Scope.DestinationID)
|
||||
}
|
||||
owners[owner.Scope] = owner
|
||||
}
|
||||
if s.Outputs == nil {
|
||||
return fmt.Errorf("state outputs is required")
|
||||
}
|
||||
seenPaths := make(map[string]struct{}, len(s.Outputs))
|
||||
for index, output := range s.Outputs {
|
||||
if err := validateSharedRootOutput(index, output, owners); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := seenPaths[output.Path]; exists {
|
||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seenPaths[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateOwnerRecord(index int, owner OwnerRecord) error {
|
||||
if owner.Scope.PipelineID == "" {
|
||||
return fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
||||
}
|
||||
if owner.Scope.DestinationID == "" {
|
||||
return fmt.Errorf("state owners[%d].destination_id is required", index)
|
||||
}
|
||||
if owner.Reconciliation.Mode != config.ReconciliationModeReplace && owner.Reconciliation.Mode != config.ReconciliationModeMerge {
|
||||
return fmt.Errorf("state owners[%d].reconciliation.mode must be %s or %s", index, config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
if err := validateEmbeddedManifest(owner.Source.Manifest); err != nil {
|
||||
return fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
||||
}
|
||||
if owner.Links != nil && owner.Links.PrimaryURL != "" {
|
||||
if err := link.ValidateHTTPURL(owner.Links.PrimaryURL); err != nil {
|
||||
return fmt.Errorf("state owners[%d].links.primary_url: %w", index, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSharedRootOutput(index int, output SharedRootOutputFile, owners map[OwnerScope]OwnerRecord) error {
|
||||
if err := storage.ValidatePath(output.Path); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
||||
}
|
||||
switch output.Kind {
|
||||
case OutputKindSource, OutputKindGenerated:
|
||||
default:
|
||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
||||
}
|
||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
||||
}
|
||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
if output.URL != "" {
|
||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
||||
}
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||
}
|
||||
if output.Size < 0 {
|
||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||
}
|
||||
if output.Owner.PipelineID == "" {
|
||||
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if output.Owner.DestinationID == "" {
|
||||
return fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
if _, exists := owners[output.Owner]; !exists {
|
||||
return fmt.Errorf("state outputs[%d] references unknown owner %s/%s", index, output.Owner.PipelineID, output.Owner.DestinationID)
|
||||
}
|
||||
if output.SourceID == "" {
|
||||
return fmt.Errorf("state outputs[%d].source_id is required", index)
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SourceDigest); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_digest: %w", index, err)
|
||||
}
|
||||
if output.SourceCreated.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].source_created is required", index)
|
||||
}
|
||||
if output.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
||||
}
|
||||
if output.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,320 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestParseDocumentHandlesSingleOwnerAndSharedRoot(t *testing.T) {
|
||||
singleOwner, err := ParseDocument([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(single owner) error = %v", err)
|
||||
}
|
||||
if singleOwner.SingleOwner == nil || singleOwner.SharedRoot != nil {
|
||||
t.Fatalf("single owner document = %#v", singleOwner)
|
||||
}
|
||||
|
||||
sharedRoot, err := ParseDocument([]byte(validSharedRootStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(shared root) error = %v", err)
|
||||
}
|
||||
if sharedRoot.SharedRoot == nil || sharedRoot.SingleOwner != nil {
|
||||
t.Fatalf("shared root document = %#v", sharedRoot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSharedRootState(t *testing.T) {
|
||||
state, err := ParseSharedRoot([]byte(validSharedRootStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseSharedRoot() error = %v", err)
|
||||
}
|
||||
if got, want := state.SchemaVersion, SharedRootSchemaVersion; got != want {
|
||||
t.Fatalf("schema version = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("created_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSharedRoot; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(state.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(state.Outputs), 2; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
manifest, ok := state.SourceManifest(scope)
|
||||
if !ok {
|
||||
t.Fatal("SourceManifest() ok = false, want true")
|
||||
}
|
||||
if got, want := manifest.ID, validManifest(t).ID; got != want {
|
||||
t.Fatalf("source manifest id = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSharedRootRejectsInvalidMetadata(t *testing.T) {
|
||||
tests := map[string]func(string) string{
|
||||
"schema": func(body string) string {
|
||||
return strings.Replace(body, `"schema_version": 3`, `"schema_version": 2`, 1)
|
||||
},
|
||||
"state mode": func(body string) string {
|
||||
return strings.Replace(body, `"mode": "shared_root"`, `"mode": "single_owner"`, 1)
|
||||
},
|
||||
"duplicate owner": func(body string) string {
|
||||
return strings.Replace(body, `"destination_id": "html"`, `"destination_id": "archive"`, 1)
|
||||
},
|
||||
"unknown output owner": func(body string) string {
|
||||
return strings.Replace(body, `"destination_id": "html",`, `"destination_id": "missing",`, 1)
|
||||
},
|
||||
"duplicate output": func(body string) string {
|
||||
return strings.Replace(body, `"path": "report.html"`, `"path": "report.md"`, 1)
|
||||
},
|
||||
"invalid source digest": func(body string) string {
|
||||
return strings.Replace(body, `"source_digest": "sha256:`, `"source_digest": "SHA256:`, 1)
|
||||
},
|
||||
"invalid owner link": func(body string) string {
|
||||
return strings.Replace(body, `"primary_url": "https://reports.example.com/archive/report.md"`, `"primary_url": "file:///tmp/report.md"`, 1)
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := ParseSharedRoot([]byte(mutate(validSharedRootStateJSON(t))))
|
||||
if err == nil {
|
||||
t.Fatal("ParseSharedRoot() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootMarshalNormalizesTimestamps(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
state.CreatedAt = time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
||||
state.UpdatedAt = state.CreatedAt
|
||||
state.Outputs[0].SourceCreated = state.CreatedAt
|
||||
state.Outputs[0].CreatedAt = state.CreatedAt
|
||||
state.Outputs[0].UpdatedAt = state.CreatedAt
|
||||
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`"created_at":"2026-05-30T11:12:00Z"`,
|
||||
`"updated_at":"2026-05-30T11:12:00Z"`,
|
||||
`"source_created":"2026-05-30T11:12:00Z"`,
|
||||
} {
|
||||
if !strings.Contains(string(data), want) {
|
||||
t.Fatalf("json = %s, want %s", data, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootOutputHelpers(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
html := CurrentOwnerScope("reports", "html")
|
||||
|
||||
if got, want := state.ManagedOutputPaths(archive), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("archive paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := state.AllManagedOutputPaths(), []string{"report.md", "report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("all paths = %#v, want %#v", got, want)
|
||||
}
|
||||
conflict, ok := state.PathOwnershipConflict(archive, []string{"report.html"})
|
||||
if !ok || conflict.Owner != html {
|
||||
t.Fatalf("conflict = %#v ok=%t, want html owner conflict", conflict, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingSharedRootOwnerOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
next, changed := RemoveMissingSharedRootOwnerOutputs(state, archive, []string{"report.md", "report.html"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingSharedRootOwnerOutputs() changed = false, want true")
|
||||
}
|
||||
if got, want := next.AllManagedOutputPaths(), []string{"report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if _, ok := next.OutputOwner("report.html"); !ok {
|
||||
t.Fatal("report.html owner missing, want unrelated owner preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingSharedRootOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
next, changed := RemoveMissingSharedRootOutputs(state, []string{"report.md", "report.html"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingSharedRootOutputs() changed = false, want true")
|
||||
}
|
||||
if got := next.AllManagedOutputPaths(); len(got) != 0 {
|
||||
t.Fatalf("paths = %#v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootProjectAndMergeOwnerOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
owner, ok := state.Owner(archive)
|
||||
if !ok {
|
||||
t.Fatal("Owner() ok = false, want true")
|
||||
}
|
||||
now := time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC)
|
||||
planned := ProjectSharedRootOutputs([]OutputProjection{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: validManifest(t).Files[0].SHA256,
|
||||
Size: validManifest(t).Files[0].Size,
|
||||
}, {
|
||||
Path: "summary.txt",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: validManifest(t).Files[1].SHA256,
|
||||
Size: validManifest(t).Files[1].Size,
|
||||
}}, state.Outputs, archive, validManifest(t), now)
|
||||
|
||||
merged, err := MergeOwnerOutputs(state, archive, owner, planned)
|
||||
if err != nil {
|
||||
t.Fatalf("MergeOwnerOutputs() error = %v", err)
|
||||
}
|
||||
if got, want := merged.AllManagedOutputPaths(), []string{"report.md", "report.html", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("merged paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if merged.Outputs[0].CreatedAt.Equal(now) {
|
||||
t.Fatalf("merged updated output created_at = %s, want preserved timestamp", merged.Outputs[0].CreatedAt)
|
||||
}
|
||||
if !merged.Outputs[0].UpdatedAt.Equal(now) {
|
||||
t.Fatalf("merged updated output updated_at = %s, want %s", merged.Outputs[0].UpdatedAt, now)
|
||||
}
|
||||
|
||||
replaced, err := ReplaceOwnerOutputs(state, archive, owner, planned)
|
||||
if err != nil {
|
||||
t.Fatalf("ReplaceOwnerOutputs() error = %v", err)
|
||||
}
|
||||
if got, want := replaced.AllManagedOutputPaths(), []string{"report.html", "report.md", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("replaced paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootOwnerOutputHelpersRejectConflicts(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
owner, ok := state.Owner(archive)
|
||||
if !ok {
|
||||
t.Fatal("Owner() ok = false, want true")
|
||||
}
|
||||
planned := []SharedRootOutputFile{{
|
||||
Path: "report.html",
|
||||
Kind: OutputKindSource,
|
||||
Owner: archive,
|
||||
CreatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
||||
UpdatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
||||
}}
|
||||
|
||||
if _, err := MergeOwnerOutputs(state, archive, owner, planned); err == nil {
|
||||
t.Fatal("MergeOwnerOutputs() error = nil, want owner conflict")
|
||||
}
|
||||
if _, err := ReplaceOwnerOutputs(state, archive, owner, planned); err == nil {
|
||||
t.Fatal("ReplaceOwnerOutputs() error = nil, want owner conflict")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareSharedRootOwnerScopesCurrentOwner(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
source := validManifest(t)
|
||||
source.Created = source.Created.Add(time.Hour)
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
|
||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{SharedRoot: &state, HasContents: true})
|
||||
if comparison.Outcome != OutcomeDestinationOlder {
|
||||
t.Fatalf("comparison = %#v, want destination older for current owner", comparison)
|
||||
}
|
||||
|
||||
missing := CompareSharedRootOwner(source, CurrentOwnerScope("missing", "archive"), DestinationStatus{SharedRoot: &state, HasContents: true})
|
||||
if missing.Outcome != OutcomeDestinationAbsent {
|
||||
t.Fatalf("missing owner comparison = %#v, want destination absent", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareSharedRootOwnerAcceptsMatchingSingleOwnerState(t *testing.T) {
|
||||
singleOwner, err := Parse([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
source := singleOwner.Source.Manifest
|
||||
source.Created = source.Created.Add(time.Hour)
|
||||
scope := CurrentOwnerScope(singleOwner.PipelineID, singleOwner.DestinationID)
|
||||
|
||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{State: &singleOwner, HasContents: true})
|
||||
if comparison.Outcome != OutcomeDestinationOlder {
|
||||
t.Fatalf("comparison = %#v, want destination older", comparison)
|
||||
}
|
||||
}
|
||||
|
||||
func validSharedRootStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(validSharedRootState(t), "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared root state: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func validSharedRootState(t *testing.T) SharedRootState {
|
||||
t.Helper()
|
||||
source := validManifest(t)
|
||||
htmlSource := source
|
||||
htmlSource.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
return SharedRootState{
|
||||
SchemaVersion: SharedRootSchemaVersion,
|
||||
DistributorVersion: "dev",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: StatePolicy{Mode: StateModeSharedRoot},
|
||||
Owners: []OwnerRecord{{
|
||||
Scope: CurrentOwnerScope("reports", "archive"),
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
||||
}, {
|
||||
Scope: CurrentOwnerScope("reports", "html"),
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
||||
Source: SourceState{Manifest: htmlSource},
|
||||
}},
|
||||
Outputs: []SharedRootOutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
Owner: CurrentOwnerScope("reports", "archive"),
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
Kind: OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
URL: "https://reports.example.com/html/report.html",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 128,
|
||||
Owner: CurrentOwnerScope("reports", "html"),
|
||||
SourceID: htmlSource.ID,
|
||||
SourceDigest: htmlSource.Digest,
|
||||
SourceCreated: htmlSource.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
OutputKindSource = "source"
|
||||
OutputKindGenerated = "generated"
|
||||
)
|
||||
|
||||
func Validate(s DistributorState) error {
|
||||
if s.SchemaVersion != SchemaVersion {
|
||||
return fmt.Errorf("state schema_version must be %d", SchemaVersion)
|
||||
}
|
||||
if s.PipelineID == "" {
|
||||
return fmt.Errorf("state pipeline_id is required")
|
||||
}
|
||||
if s.DestinationID == "" {
|
||||
return fmt.Errorf("state destination_id is required")
|
||||
}
|
||||
if s.PublishedAt.IsZero() {
|
||||
return fmt.Errorf("state published_at is required")
|
||||
}
|
||||
if s.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state created_at is required")
|
||||
}
|
||||
if s.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state updated_at is required")
|
||||
}
|
||||
if s.State.Mode != StateModeSingleOwner {
|
||||
return fmt.Errorf("state state.mode must be %s", StateModeSingleOwner)
|
||||
}
|
||||
if s.Reconciliation.Mode != config.ReconciliationModeReplace && s.Reconciliation.Mode != config.ReconciliationModeMerge {
|
||||
return fmt.Errorf("state reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
|
||||
return fmt.Errorf("state source.manifest: %w", err)
|
||||
}
|
||||
if s.Links != nil && s.Links.PrimaryURL != "" {
|
||||
if err := link.ValidateHTTPURL(s.Links.PrimaryURL); err != nil {
|
||||
return fmt.Errorf("state links.primary_url: %w", err)
|
||||
}
|
||||
}
|
||||
if s.Outputs == nil {
|
||||
return fmt.Errorf("state outputs is required")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(s.Outputs))
|
||||
for index, output := range s.Outputs {
|
||||
if err := validateOutput(index, output); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := seen[output.Path]; exists {
|
||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seen[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateEmbeddedManifest(manifest bundle.Manifest) error {
|
||||
return bundle.ValidateManifest(manifest)
|
||||
}
|
||||
|
||||
func validateOutput(index int, output OutputFile) error {
|
||||
if err := storage.ValidatePath(output.Path); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
||||
}
|
||||
switch output.Kind {
|
||||
case OutputKindSource, OutputKindGenerated:
|
||||
default:
|
||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
||||
}
|
||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
||||
}
|
||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
if output.URL != "" {
|
||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
||||
}
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||
}
|
||||
if output.Size < 0 {
|
||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||
}
|
||||
if output.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
||||
}
|
||||
if output.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -11,8 +11,6 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
)
|
||||
@@ -33,13 +31,6 @@ type BundleOptions struct {
|
||||
ExtraFiles []SourceFile
|
||||
}
|
||||
|
||||
type DestinationStateOptions struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
DistributorVersion string
|
||||
PublishedAt time.Time
|
||||
}
|
||||
|
||||
func DefaultSourceFiles() []SourceFile {
|
||||
return []SourceFile{
|
||||
{Path: "report.md", Data: "# Report\nSunny.\n"},
|
||||
@@ -133,28 +124,6 @@ func AssertFakeMissing(t testing.TB, backend *fake.Backend, path string) {
|
||||
}
|
||||
}
|
||||
|
||||
func WriteFakeDestinationState(t testing.TB, backend *fake.Backend, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||
t.Helper()
|
||||
destinationState := DestinationState(manifest, opts)
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal destination state: %v", err)
|
||||
}
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||
for _, output := range destinationState.Outputs {
|
||||
path, err := storage.Join(relative, output.Path)
|
||||
if err != nil {
|
||||
t.Fatalf("join output path: %v", err)
|
||||
}
|
||||
WriteFakeFile(t, backend, path, "old")
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
@@ -318,47 +287,6 @@ pipelines:
|
||||
`)
|
||||
}
|
||||
|
||||
func WriteDestinationState(t testing.TB, root, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||
t.Helper()
|
||||
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
|
||||
if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
|
||||
t.Fatalf("mkdir destination bundle: %v", err)
|
||||
}
|
||||
destinationState := DestinationState(manifest, opts)
|
||||
writeJSONFile(t, filepath.Join(bundleRoot, storage.StateFileName), destinationState)
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func DestinationState(manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||
publishedAt := defaultPublishedAt(opts.PublishedAt)
|
||||
return state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
DistributorVersion: opts.DistributorVersion,
|
||||
PipelineID: defaultString(opts.PipelineID, "reports"),
|
||||
DestinationID: defaultString(opts.DestinationID, "archive"),
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
Outputs: sourceOutputs(manifest, publishedAt),
|
||||
}
|
||||
}
|
||||
|
||||
func ReadDestinationState(t testing.TB, path string) state.DistributorState {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read destination state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse destination state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func AssertFile(t testing.TB, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
@@ -392,22 +320,6 @@ func sourceFiles(opts BundleOptions) []SourceFile {
|
||||
return files
|
||||
}
|
||||
|
||||
func sourceOutputs(manifest bundle.Manifest, publishedAt time.Time) []state.OutputFile {
|
||||
outputs := make([]state.OutputFile, 0, len(manifest.Files))
|
||||
for _, file := range manifest.Files {
|
||||
outputs = append(outputs, state.OutputFile{
|
||||
Path: file.Path,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: file.Path,
|
||||
SHA256: file.SHA256,
|
||||
Size: file.Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
})
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
|
||||
func writeConfigFile(t testing.TB, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
@@ -455,10 +367,3 @@ func defaultCreated(value time.Time) time.Time {
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func defaultPublishedAt(value time.Time) time.Time {
|
||||
if value.IsZero() {
|
||||
return time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
@@ -123,7 +123,7 @@ func (t *Transformer) render(ctx context.Context, req transform.Request, sourceF
|
||||
if err := t.renderer.Convert(data, &rendered); err != nil {
|
||||
return nil, fmt.Errorf("render markdown source %q: %w", sourceFile, err)
|
||||
}
|
||||
return wrapHTML(rendered.Bytes()), nil
|
||||
return wrapHTML(rendered.Bytes(), req.Markdown.CssHref), nil
|
||||
}
|
||||
|
||||
func markdownMode(mode string) string {
|
||||
|
||||
@@ -40,6 +40,43 @@ func TestGenerateMarkdownSidecar(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkdownWithoutCSSHrefPreservesWrapper(t *testing.T) {
|
||||
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
|
||||
|
||||
outputs, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
want := "<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<title></title>\n</head>\n<body>\n<h1>Title</h1>\n<p>Hello.</p>\n</body>\n</html>\n"
|
||||
if got := string(outputs[0].Data); got != want {
|
||||
t.Fatalf("html = %q, want existing wrapper %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkdownSidecarWithCSSHref(t *testing.T) {
|
||||
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
|
||||
|
||||
outputs, err := New().Generate(context.Background(), transform.Request{
|
||||
SourceBackend: backend,
|
||||
SourceBundle: sourceBundle,
|
||||
Markdown: transform.MarkdownOptions{CssHref: "/assets/report.css?v=1&theme=main"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
output := outputs[0]
|
||||
html := string(output.Data)
|
||||
wantLink := "<meta charset=\"utf-8\">\n<link rel=\"stylesheet\" href=\"/assets/report.css?v=1&theme=main\">\n<title></title>"
|
||||
if !strings.Contains(html, wantLink) {
|
||||
t.Fatalf("html = %q, want stylesheet link %q", html, wantLink)
|
||||
}
|
||||
if output.SHA256 != bundle.FileDigest(output.Data) || output.Size != int64(len(output.Data)) {
|
||||
t.Fatalf("digest/size metadata = %s/%d", output.SHA256, output.Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkdownIndexExplicitInput(t *testing.T) {
|
||||
backend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, backend, "", testutil.BundleOptions{
|
||||
@@ -74,6 +111,29 @@ func TestGenerateMarkdownIndexExplicitInput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkdownIndexWithCSSHref(t *testing.T) {
|
||||
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
|
||||
|
||||
outputs, err := New().Generate(context.Background(), transform.Request{
|
||||
SourceBackend: backend,
|
||||
SourceBundle: sourceBundle,
|
||||
Markdown: transform.MarkdownOptions{
|
||||
Mode: transform.MarkdownModeIndex,
|
||||
CssHref: "https://example.com/assets/report.css",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
if got, want := outputs[0].Path, "index.html"; got != want {
|
||||
t.Fatalf("path = %q, want %q", got, want)
|
||||
}
|
||||
if !strings.Contains(string(outputs[0].Data), `<link rel="stylesheet" href="https://example.com/assets/report.css">`) {
|
||||
t.Fatalf("html = %q, want stylesheet link", outputs[0].Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkdownIndexSelectsOnlyMarkdownFile(t *testing.T) {
|
||||
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
|
||||
|
||||
|
||||
@@ -1,10 +1,19 @@
|
||||
package markdown
|
||||
|
||||
import "bytes"
|
||||
import (
|
||||
"bytes"
|
||||
"html"
|
||||
)
|
||||
|
||||
func wrapHTML(body []byte) []byte {
|
||||
func wrapHTML(body []byte, cssHref string) []byte {
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString("<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<title></title>\n</head>\n<body>\n")
|
||||
buf.WriteString("<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n")
|
||||
if cssHref != "" {
|
||||
buf.WriteString("<link rel=\"stylesheet\" href=\"")
|
||||
buf.WriteString(html.EscapeString(cssHref))
|
||||
buf.WriteString("\">\n")
|
||||
}
|
||||
buf.WriteString("<title></title>\n</head>\n<body>\n")
|
||||
buf.Write(body)
|
||||
buf.WriteString("</body>\n</html>\n")
|
||||
return buf.Bytes()
|
||||
|
||||
@@ -25,6 +25,7 @@ type Request struct {
|
||||
type MarkdownOptions struct {
|
||||
Mode string
|
||||
Input string
|
||||
CssHref string
|
||||
}
|
||||
|
||||
type Transformer interface {
|
||||
|
||||
Reference in New Issue
Block a user