Compare commits
5 Commits
4e673dda76
...
8366af6fb6
| Author | SHA1 | Date | |
|---|---|---|---|
| 8366af6fb6 | |||
| 5e47d89355 | |||
| 2e0d903626 | |||
| 484fda2514 | |||
| 9a2eaf8e5e |
11
docs/cli.md
11
docs/cli.md
@@ -72,7 +72,7 @@ distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
|
|||||||
|
|
||||||
- `--config <path>` loads the pipeline configuration. If omitted, the application uses `/usr/local/etc/distributor/config.yml`.
|
- `--config <path>` loads the pipeline configuration. If omitted, the application uses `/usr/local/etc/distributor/config.yml`.
|
||||||
- `--dry-run` validates inputs and reports destination actions without applying changes.
|
- `--dry-run` validates inputs and reports destination actions without applying changes.
|
||||||
- `--force` permits a run when destination state indicates a conservative safety check would otherwise block it.
|
- `--force` permits exceptional catalog replacement when a dry run reports `force_replace` for unmanaged content, a planned unmanaged path collision, invalid state, or unsupported future state.
|
||||||
- `--format text|json` selects human-readable or machine-readable output.
|
- `--format text|json` selects human-readable or machine-readable output.
|
||||||
|
|
||||||
`run` accepts no positional arguments.
|
`run` accepts no positional arguments.
|
||||||
@@ -86,7 +86,7 @@ distributor reconcile-state --config <path> --pipeline <id> --destination <id> [
|
|||||||
- `--config <path>` loads the pipeline configuration and is required.
|
- `--config <path>` loads the pipeline configuration and is required.
|
||||||
- `--pipeline <id>` selects the pipeline used to identify the destination root and is required.
|
- `--pipeline <id>` selects the pipeline used to identify the destination root and is required.
|
||||||
- `--destination <id>` selects the destination root and is required.
|
- `--destination <id>` selects the destination root and is required.
|
||||||
- `--all-owners` repairs missing managed output records for every owner in a shared-root state file. Without it, shared-root repair is scoped to the selected pipeline and destination owner.
|
- `--all-owners` repairs missing managed output records for every owner in the selected catalog state file. Without it, repair is scoped to the selected pipeline and destination owner.
|
||||||
- `--dry-run` reports repairs without rewriting `.distributor.json`.
|
- `--dry-run` reports repairs without rewriting `.distributor.json`.
|
||||||
- `--format text|json` selects human-readable or machine-readable output.
|
- `--format text|json` selects human-readable or machine-readable output.
|
||||||
|
|
||||||
@@ -206,7 +206,7 @@ go run ./cmd/distributor run --config examples/local-publish.yml --dry-run
|
|||||||
go run ./cmd/distributor run --config examples/local-publish.yml
|
go run ./cmd/distributor run --config examples/local-publish.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `--format json` when automation needs structured run results. Use `--force` only when the operator has reviewed the destination state conflict and intentionally wants to continue.
|
Use `--format json` when automation needs structured run results. Use `--force` only after `--dry-run --force` reports the intended bounded `force_replace` action.
|
||||||
|
|
||||||
### Repair Destination State Records
|
### Repair Destination State Records
|
||||||
|
|
||||||
@@ -229,7 +229,7 @@ go run ./cmd/distributor reconcile-state \
|
|||||||
--destination local-archive
|
--destination local-archive
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `--all-owners` only for shared-root destination state when all owners inside the selected root should be repaired.
|
Use `--all-owners` only when every owner inside the selected catalog root should be repaired.
|
||||||
|
|
||||||
### Prune Managed Outputs
|
### Prune Managed Outputs
|
||||||
|
|
||||||
@@ -292,7 +292,8 @@ Text output is optimized for direct operator use. JSON output is optimized for a
|
|||||||
- Usage errors and fatal setup errors exit non-zero and do not emit a JSON result document.
|
- Usage errors and fatal setup errors exit non-zero and do not emit a JSON result document.
|
||||||
- `run --format json` emits a JSON result for partial destination failures, sets `ok` to `false`, includes result details and errors, and exits non-zero.
|
- `run --format json` emits a JSON result for partial destination failures, sets `ok` to `false`, includes result details and errors, and exits non-zero.
|
||||||
- Warnings are included in JSON output and are printed in text output when relevant.
|
- Warnings are included in JSON output and are printed in text output when relevant.
|
||||||
- `run` summaries include separate `replace_older`, `replace_conflict`, `replace_newer`, `replace_takeover`, and `force_replace` counters. Takeover destination actions include `takeover_mode` in JSON and text output.
|
- `run` summaries include `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and `fail_conflict` counters. Destination action records use the same stable action values and include the resolved `destination_path`.
|
||||||
|
- `skip_same` means the planned outputs already match valid catalog metadata, so `run` does not write outputs, rewrite `.distributor.json`, delete files, or notify. The decision is based on catalog metadata and does not read destination file bytes.
|
||||||
|
|
||||||
## Diagnostics And Recovery
|
## Diagnostics And Recovery
|
||||||
|
|
||||||
|
|||||||
224
docs/config.md
224
docs/config.md
@@ -2,11 +2,11 @@
|
|||||||
|
|
||||||
Audience: administrators, operators, and advanced users who write YAML configuration for `distributor`.
|
Audience: administrators, operators, and advanced users who write YAML configuration for `distributor`.
|
||||||
|
|
||||||
This document is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operating procedures live in [Operations](operations.md), symptom-oriented recovery lives in [Troubleshooting](troubleshooting.md), and external contracts live under [Integrations](integrations/source-bundle.md).
|
This is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operations guidance lives in [Operations](operations.md), recovery guidance lives in [Troubleshooting](troubleshooting.md), and file-format contracts live under [Integrations](integrations/source-bundle.md).
|
||||||
|
|
||||||
## Config File Loading
|
## Config File Loading
|
||||||
|
|
||||||
`distributor run --config <path>` and `distributor serve --config <path>` load the YAML file at `<path>`. If `--config` is omitted, both commands use:
|
`distributor run --config <path>` and `distributor serve --config <path>` load the YAML file at `<path>`. If `--config` is omitted, commands use:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
/usr/local/etc/distributor/config.yml
|
/usr/local/etc/distributor/config.yml
|
||||||
@@ -21,7 +21,7 @@ Runtime backend support is command-specific:
|
|||||||
- `serve` uses `http_upload` sources through the HTTP upload API and publishes to configured `local`, `ssh`, and `s3` destinations.
|
- `serve` uses `http_upload` sources through the HTTP upload API and publishes to configured `local`, `ssh`, and `s3` destinations.
|
||||||
- `http_upload` is valid only as a source backend.
|
- `http_upload` is valid only as a source backend.
|
||||||
|
|
||||||
## Minimal Local Config
|
## Minimal Working Config
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
pipelines:
|
pipelines:
|
||||||
@@ -35,9 +35,9 @@ pipelines:
|
|||||||
path: /srv/reports/archive
|
path: /srv/reports/archive
|
||||||
```
|
```
|
||||||
|
|
||||||
This config publishes source files only. It uses default validation, destination path mapping, publish, state, reconciliation, retention, transfer, and HTTP server values.
|
This publishes source files only. It uses default validation, additive workflow, preserve-relative path mapping, source-only publish policy, disabled pruning, and default HTTP server values.
|
||||||
|
|
||||||
## Production-Oriented Local Config
|
## Production-Oriented Config
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
server:
|
server:
|
||||||
@@ -61,23 +61,20 @@ pipelines:
|
|||||||
- id: archive
|
- id: archive
|
||||||
backend: local
|
backend: local
|
||||||
path: /srv/reports/archive
|
path: /srv/reports/archive
|
||||||
|
workflow: additive
|
||||||
publish:
|
publish:
|
||||||
source: true
|
source: true
|
||||||
html: false
|
html: false
|
||||||
path_mapping:
|
path_mapping:
|
||||||
mode: preserve_relative
|
mode: preserve_relative
|
||||||
reconciliation:
|
retention:
|
||||||
mode: replace
|
prune:
|
||||||
transfer:
|
enabled: false
|
||||||
on_destination_same: skip
|
|
||||||
on_destination_older: replace
|
|
||||||
on_destination_newer: skip
|
|
||||||
on_conflict: fail
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## HTTP Upload Source Config
|
## HTTP Upload Source Config
|
||||||
|
|
||||||
HTTP upload sources are configured on pipelines and are served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML.
|
HTTP upload sources are configured on pipelines and served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
server:
|
server:
|
||||||
@@ -97,15 +94,13 @@ pipelines:
|
|||||||
- id: weather-daily
|
- id: weather-daily
|
||||||
source:
|
source:
|
||||||
backend: http_upload
|
backend: http_upload
|
||||||
staging_path: /var/spool/distributor/weather-daily
|
|
||||||
max_upload_size: 20MB
|
|
||||||
destinations:
|
destinations:
|
||||||
- id: archive
|
- id: archive
|
||||||
backend: local
|
backend: local
|
||||||
path: /srv/reports/archive
|
path: /srv/reports/archive
|
||||||
```
|
```
|
||||||
|
|
||||||
`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to. One token may authorize multiple upload pipelines, and multiple tokens may authorize the same upload pipeline.
|
`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to.
|
||||||
|
|
||||||
For `http_upload` sources, `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`. `max_upload_size` defaults to `server.http.max_upload_size`.
|
For `http_upload` sources, `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`. `max_upload_size` defaults to `server.http.max_upload_size`.
|
||||||
|
|
||||||
@@ -136,7 +131,7 @@ See [Secrets](#secrets) for resolution rules.
|
|||||||
|
|
||||||
Each token has:
|
Each token has:
|
||||||
|
|
||||||
- `id`: required unique slug-like identifier for the token record. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
|
- `id`: required unique slug-like identifier for the token record.
|
||||||
- `token_env`: required environment variable or secret-file name containing the bearer token value.
|
- `token_env`: required environment variable or secret-file name containing the bearer token value.
|
||||||
- `allow_pipelines`: required non-empty list of configured pipeline ids whose source backend is `http_upload`.
|
- `allow_pipelines`: required non-empty list of configured pipeline ids whose source backend is `http_upload`.
|
||||||
|
|
||||||
@@ -148,12 +143,12 @@ Token values must resolve to non-empty strings and must be unique across token r
|
|||||||
|
|
||||||
Each pipeline has:
|
Each pipeline has:
|
||||||
|
|
||||||
- `id`: required unique slug-like identifier. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
|
- `id`: required unique slug-like identifier.
|
||||||
- `source`: required source backend config.
|
- `source`: required source backend config.
|
||||||
- `validation`: optional validation policy.
|
- `validation`: optional validation policy.
|
||||||
- `destinations`: required non-empty destination list.
|
- `destinations`: required non-empty destination list.
|
||||||
|
|
||||||
Pipeline ids must be unique across the config.
|
Slug-like identifiers must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
|
||||||
|
|
||||||
## Backend Reference
|
## Backend Reference
|
||||||
|
|
||||||
@@ -171,7 +166,7 @@ path: /srv/distributor/archive
|
|||||||
|
|
||||||
### SSH/SFTP Backend
|
### SSH/SFTP Backend
|
||||||
|
|
||||||
SSH backends use native SFTP and can be used as sources and destinations. Adapter protocol behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md).
|
SSH backends use native SFTP and can be used as sources and destinations. Adapter behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md).
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backend: ssh
|
backend: ssh
|
||||||
@@ -193,17 +188,11 @@ host_key_policy: strict
|
|||||||
- `known_hosts`: optional OpenSSH `known_hosts` path.
|
- `known_hosts`: optional OpenSSH `known_hosts` path.
|
||||||
- `host_key_policy`: optional host key policy. Default: `accept-new`.
|
- `host_key_policy`: optional host key policy. Default: `accept-new`.
|
||||||
|
|
||||||
Accepted host key policy values:
|
Accepted host key policy values are `strict` or boolean `true`, `accept-new`, and `off` or boolean `false`. Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML.
|
||||||
|
|
||||||
- `strict` or boolean `true`: require a matching known host key.
|
|
||||||
- `accept-new`: accept and persist a new host key, but reject changed known keys.
|
|
||||||
- `off` or boolean `false`: disable host key checking.
|
|
||||||
|
|
||||||
Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML.
|
|
||||||
|
|
||||||
### S3-Compatible Backend
|
### S3-Compatible Backend
|
||||||
|
|
||||||
S3 backends can be used as sources and destinations. Adapter protocol behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md).
|
S3 backends can be used as sources and destinations. Adapter behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md).
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backend: s3
|
backend: s3
|
||||||
@@ -255,42 +244,50 @@ Source bundle digest mismatches fail validation before destination writes occur.
|
|||||||
|
|
||||||
## Destination Fields
|
## Destination Fields
|
||||||
|
|
||||||
Each destination embeds a backend config at the destination level and may also configure publishing, transforms, path mapping, links, state, reconciliation, retention, and transfer behavior.
|
Each destination embeds a backend config at the destination level and may also configure workflow, publishing, transforms, path mapping, links, and retention.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
destinations:
|
destinations:
|
||||||
- id: archive
|
- id: archive
|
||||||
backend: local
|
backend: local
|
||||||
path: /srv/reports/archive
|
path: /srv/reports/archive
|
||||||
|
workflow: additive
|
||||||
publish:
|
publish:
|
||||||
source: true
|
source: true
|
||||||
html: false
|
html: false
|
||||||
path_mapping:
|
path_mapping:
|
||||||
mode: preserve_relative
|
mode: preserve_relative
|
||||||
state:
|
|
||||||
mode: single_owner
|
|
||||||
reconciliation:
|
|
||||||
mode: replace
|
|
||||||
transfer:
|
|
||||||
on_destination_same: skip
|
|
||||||
on_destination_older: replace
|
|
||||||
on_destination_newer: skip
|
|
||||||
on_conflict: fail
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- `id`: required unique slug-like identifier within the pipeline.
|
- `id`: required unique slug-like identifier within the pipeline.
|
||||||
- Backend fields: required according to the selected destination backend.
|
- Backend fields: required according to the selected destination backend.
|
||||||
|
- `workflow`: optional catalog update workflow. Default: `additive`.
|
||||||
- `publish`: optional publish policy. Default: source-only publication.
|
- `publish`: optional publish policy. Default: source-only publication.
|
||||||
- `transform`: required only when publishing generated HTML.
|
- `transform`: required only when publishing generated HTML.
|
||||||
- `path_mapping`: optional destination path mapping policy.
|
- `path_mapping`: optional destination path mapping policy.
|
||||||
- `links`: optional public URL metadata policy.
|
- `links`: optional public URL metadata policy.
|
||||||
- `state`: optional destination state ownership policy.
|
|
||||||
- `reconciliation`: optional managed-output reconciliation policy.
|
|
||||||
- `retention`: optional managed-output retention policy.
|
- `retention`: optional managed-output retention policy.
|
||||||
- `transfer`: optional destination comparison action policy.
|
|
||||||
|
|
||||||
Destination ids must be unique within a pipeline.
|
Destination ids must be unique within a pipeline.
|
||||||
|
|
||||||
|
Pre-workflow destination policy keys for state mode, conflict handling,
|
||||||
|
ownership adoption, and per-comparison copy decisions are not accepted config
|
||||||
|
fields. YAML files containing those keys fail during config loading.
|
||||||
|
|
||||||
|
## Destination Workflow
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
workflow: additive
|
||||||
|
```
|
||||||
|
|
||||||
|
- `workflow`: optional. Accepted values are `additive` and `replacement`; default is `additive`.
|
||||||
|
|
||||||
|
`additive` writes planned outputs into the catalog and retains unrelated catalog-managed outputs in the same destination bundle path. Existing catalog records for planned paths are replaced by the current publication. A planned path that exists in storage but is not recorded in valid catalog state fails as unmanaged content unless `run --force` selects `force_replace`.
|
||||||
|
|
||||||
|
`replacement` writes planned outputs for the current pipeline and destination, and removes catalog outputs owned by the same pipeline and destination when those outputs are omitted from the new plan. Outputs owned by other pipeline/destination pairs remain catalog-managed. Replacement workflow is normal managed behavior and does not require `--force`.
|
||||||
|
|
||||||
|
Use `replacement` for stable latest-style destinations where the current owner should publish exactly the currently planned output set. Use `additive` when a destination root intentionally accumulates outputs over time or receives disjoint outputs from multiple configured destinations.
|
||||||
|
|
||||||
## Publish And Transform Policy
|
## Publish And Transform Policy
|
||||||
|
|
||||||
### Source-Only Publication
|
### Source-Only Publication
|
||||||
@@ -365,113 +362,7 @@ Primary URL policies:
|
|||||||
- `html`: use the first generated HTML output.
|
- `html`: use the first generated HTML output.
|
||||||
- `source`: use the first copied source output.
|
- `source`: use the first copied source output.
|
||||||
|
|
||||||
If no output matches the primary policy, per-output URLs may still be recorded and the top-level primary URL is omitted.
|
If no output matches the primary policy, per-output URLs may still be recorded and no primary URL is reported for the run.
|
||||||
|
|
||||||
## Destination State Policy
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
state:
|
|
||||||
mode: single_owner
|
|
||||||
```
|
|
||||||
|
|
||||||
- `state.mode`: optional. Accepted values are `single_owner` and `shared_root`; default is `single_owner`.
|
|
||||||
|
|
||||||
`single_owner` state records one pipeline/destination owner for each destination bundle path and is the default state mode.
|
|
||||||
|
|
||||||
`shared_root` records multiple pipeline/destination owners in one destination root. Publish execution preserves unrelated owners, rejects path ownership conflicts, and writes shared-root destination state.
|
|
||||||
|
|
||||||
Use `shared_root` when multiple configured destinations intentionally write disjoint output paths into the same backend root:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
pipelines:
|
|
||||||
- id: reports-source
|
|
||||||
source:
|
|
||||||
backend: local
|
|
||||||
path: /var/spool/distributor/reports
|
|
||||||
destinations:
|
|
||||||
- id: shared-root
|
|
||||||
backend: local
|
|
||||||
path: /srv/reports/shared
|
|
||||||
state:
|
|
||||||
mode: shared_root
|
|
||||||
publish:
|
|
||||||
source: true
|
|
||||||
html: false
|
|
||||||
- id: reports-html
|
|
||||||
source:
|
|
||||||
backend: local
|
|
||||||
path: /var/spool/distributor/reports
|
|
||||||
destinations:
|
|
||||||
- id: shared-root
|
|
||||||
backend: local
|
|
||||||
path: /srv/reports/shared
|
|
||||||
state:
|
|
||||||
mode: shared_root
|
|
||||||
publish:
|
|
||||||
source: false
|
|
||||||
html: true
|
|
||||||
transform:
|
|
||||||
markdown_to_html:
|
|
||||||
enabled: true
|
|
||||||
mode: sidecar
|
|
||||||
```
|
|
||||||
|
|
||||||
Every output path in a shared root belongs to exactly one `pipeline_id` and `destination_id`. A different owner planning the same path fails as a conflict unless `takeover.mode` allows the managed output path to move to the current owner.
|
|
||||||
|
|
||||||
## Takeover Policy
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
takeover:
|
|
||||||
mode: same_pipeline
|
|
||||||
```
|
|
||||||
|
|
||||||
- `takeover.mode`: optional. Accepted values are `same_pipeline`, `same_source`, `any_managed`, and `never`; default is `same_pipeline`.
|
|
||||||
|
|
||||||
Takeover controls when a destination may normally replace valid distributor-managed state whose pipeline, destination, source identity, or shared-root output owner differs from the current publication.
|
|
||||||
|
|
||||||
- `same_pipeline`: replace managed state owned by the same pipeline. The previous destination id and source id may differ.
|
|
||||||
- `same_source`: replace managed state only when the existing source manifest id matches the current source id.
|
|
||||||
- `any_managed`: replace any valid distributor-managed state at the selected destination bundle path.
|
|
||||||
- `never`: do not replace identity or source conflicts without the explicit forced replacement workflow.
|
|
||||||
|
|
||||||
For shared-root state, `same_pipeline` permits taking over output paths owned by another destination in the same pipeline, `same_source` permits taking over output paths whose owner records the same source manifest id, and `any_managed` permits taking over output paths owned by any valid shared-root owner. Unrelated owner records and non-conflicting outputs remain managed by their existing owners.
|
|
||||||
|
|
||||||
Takeover does not apply to unmanaged content, invalid destination state, same-created digest conflicts, or same-source destination-newer comparisons.
|
|
||||||
|
|
||||||
## Reconciliation Policy
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
reconciliation:
|
|
||||||
mode: replace
|
|
||||||
```
|
|
||||||
|
|
||||||
- `reconciliation.mode`: optional. Accepted values are `replace` and `merge`; default is `replace`.
|
|
||||||
|
|
||||||
Reconciliation controls how a destination with older managed state is updated after transfer policy selects `replace_older`.
|
|
||||||
|
|
||||||
`replace` deletes the prior managed outputs recorded in `.distributor.json`, deletes the state file, verifies the destination bundle path is empty, then writes only the newly planned outputs and a new state file. This is the default and is appropriate when each publication should exactly match the current publish and transform policy.
|
|
||||||
|
|
||||||
`merge` keeps prior managed outputs that are not produced by the new plan. Planned paths already recorded in existing state may be overwritten; planned paths that already exist in storage but are not recorded as managed fail as unmanaged collisions. The resulting state file records the cumulative managed output set.
|
|
||||||
|
|
||||||
Example merge destination:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
destinations:
|
|
||||||
- id: static-site
|
|
||||||
backend: local
|
|
||||||
path: /srv/reports/site
|
|
||||||
publish:
|
|
||||||
source: false
|
|
||||||
html: true
|
|
||||||
transform:
|
|
||||||
markdown_to_html:
|
|
||||||
enabled: true
|
|
||||||
mode: sidecar
|
|
||||||
reconciliation:
|
|
||||||
mode: merge
|
|
||||||
```
|
|
||||||
|
|
||||||
`links.primary_url` is selected from the newly planned outputs for the current run. Retained outputs keep their prior output metadata and timestamps.
|
|
||||||
|
|
||||||
## Retention Policy
|
## Retention Policy
|
||||||
|
|
||||||
@@ -489,28 +380,9 @@ retention:
|
|||||||
|
|
||||||
When `retention.prune.enabled` is `true`, at least one of `older_than` or `keep_latest` is required. `older_than` must be greater than zero, and `keep_latest` must be zero or greater.
|
When `retention.prune.enabled` is `true`, at least one of `older_than` or `keep_latest` is required. `older_than` must be greater than zero, and `keep_latest` must be zero or greater.
|
||||||
|
|
||||||
Pruning uses managed output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs.
|
Pruning uses catalog output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs.
|
||||||
|
|
||||||
The `prune` command is owner-scoped for shared-root state. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`.
|
The `prune` command is scoped to the selected pipeline and destination owner. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`.
|
||||||
|
|
||||||
## Transfer Policy
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
transfer:
|
|
||||||
on_destination_same: skip
|
|
||||||
on_destination_older: replace
|
|
||||||
on_destination_newer: skip
|
|
||||||
on_conflict: fail
|
|
||||||
```
|
|
||||||
|
|
||||||
Transfer fields and accepted values:
|
|
||||||
|
|
||||||
- `transfer.on_destination_same`: `skip` or `fail`. Default: `skip`.
|
|
||||||
- `transfer.on_destination_older`: `replace` or `fail`. Default: `replace`.
|
|
||||||
- `transfer.on_destination_newer`: `skip`, `replace`, or `fail`. Default: `skip`.
|
|
||||||
- `transfer.on_conflict`: `fail` or `replace`. Default: `fail`.
|
|
||||||
|
|
||||||
`replace` for `on_destination_newer` and `on_conflict` applies only to valid distributor-managed destination state. Unmanaged destination content and invalid destination state still require the explicit `run --force` workflow where supported.
|
|
||||||
|
|
||||||
## Size And Duration Values
|
## Size And Duration Values
|
||||||
|
|
||||||
@@ -543,18 +415,12 @@ Defaults are applied after YAML decoding and before validation:
|
|||||||
- S3 `force_path_style: true`
|
- S3 `force_path_style: true`
|
||||||
- `http_upload` source `staging_path: <server.http.staging_root>/<pipeline id>`
|
- `http_upload` source `staging_path: <server.http.staging_root>/<pipeline id>`
|
||||||
- `http_upload` source `max_upload_size: server.http.max_upload_size`
|
- `http_upload` source `max_upload_size: server.http.max_upload_size`
|
||||||
|
- `workflow: additive`
|
||||||
- `publish.source: true` and `publish.html: false`
|
- `publish.source: true` and `publish.html: false`
|
||||||
- `transform.markdown_to_html.mode: sidecar` when a Markdown transform block is present and mode is omitted
|
- `transform.markdown_to_html.mode: sidecar` when a Markdown transform block is present and mode is omitted
|
||||||
- `path_mapping.mode: preserve_relative`
|
- `path_mapping.mode: preserve_relative`
|
||||||
- `links.primary: auto` when a `links` block is present and `primary` is omitted
|
- `links.primary: auto` when a `links` block is present and `primary` is omitted
|
||||||
- `state.mode: single_owner`
|
|
||||||
- `reconciliation.mode: replace`
|
|
||||||
- `takeover.mode: same_pipeline`
|
|
||||||
- `retention.prune.enabled: false`
|
- `retention.prune.enabled: false`
|
||||||
- `transfer.on_destination_same: skip`
|
|
||||||
- `transfer.on_destination_older: replace`
|
|
||||||
- `transfer.on_destination_newer: skip`
|
|
||||||
- `transfer.on_conflict: fail`
|
|
||||||
|
|
||||||
## Secrets
|
## Secrets
|
||||||
|
|
||||||
@@ -577,7 +443,7 @@ Fields resolved through this resolver:
|
|||||||
|
|
||||||
## Maintained Examples
|
## Maintained Examples
|
||||||
|
|
||||||
Maintained examples live under [examples](../examples/). Config tests load every file listed here.
|
Maintained examples live under [examples](../examples/). Config tests load these YAML files.
|
||||||
|
|
||||||
Local examples:
|
Local examples:
|
||||||
|
|
||||||
@@ -587,8 +453,8 @@ Local examples:
|
|||||||
- `local-index.yml`: local `index.html` publication.
|
- `local-index.yml`: local `index.html` publication.
|
||||||
- `fan-out.yml`: local fan-out publication to source and HTML destinations.
|
- `fan-out.yml`: local fan-out publication to source and HTML destinations.
|
||||||
- `archive-and-latest.yml`: local archive plus fixed latest publication.
|
- `archive-and-latest.yml`: local archive plus fixed latest publication.
|
||||||
- `merge-reconciliation.yml`: local HTML publication using merge reconciliation.
|
- `additive-workflow.yml`: two destinations publishing disjoint outputs into one catalog-managed root.
|
||||||
- `shared-root.yml`: two local pipelines publishing disjoint outputs into one shared destination root.
|
- `replacement-workflow.yml`: fixed-path replacement workflow for a stable latest-style output set.
|
||||||
- `http-upload-local.yml`: local HTTP upload server config; requires `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` in the process environment or as a secret-file name before running `serve`.
|
- `http-upload-local.yml`: local HTTP upload server config; requires `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` in the process environment or as a secret-file name before running `serve`.
|
||||||
|
|
||||||
Environment-gated remote examples:
|
Environment-gated remote examples:
|
||||||
|
|||||||
@@ -2,197 +2,37 @@
|
|||||||
|
|
||||||
Audience: operators, integrators, and maintainers who inspect or reason about destination `.distributor.json` files.
|
Audience: operators, integrators, and maintainers who inspect or reason about destination `.distributor.json` files.
|
||||||
|
|
||||||
Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for comparison, skip, replacement, and recovery decisions.
|
Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for catalog planning, managed replacement, retention pruning, repair, and recovery.
|
||||||
|
|
||||||
## Single-Owner State Schema
|
## Catalog State Schema
|
||||||
|
|
||||||
State written by `run` for `state.mode: single_owner` uses schema version `2`.
|
Publish execution writes catalog state with `schema_version` `4`.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"schema_version": 2,
|
"schema_version": 4,
|
||||||
"distributor_version": "dev",
|
|
||||||
"pipeline_id": "reports",
|
|
||||||
"destination_id": "archive",
|
|
||||||
"published_at": "2026-06-04T12:00:00Z",
|
|
||||||
"created_at": "2026-06-04T12:00:00Z",
|
|
||||||
"updated_at": "2026-06-04T12:00:00Z",
|
|
||||||
"state": {
|
|
||||||
"mode": "single_owner"
|
|
||||||
},
|
|
||||||
"reconciliation": {
|
|
||||||
"mode": "replace"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"manifest": {
|
|
||||||
"schema_version": 1,
|
|
||||||
"id": "reports.example.2026-06-04",
|
|
||||||
"digest": "sha256:...",
|
|
||||||
"created": "2026-06-04T11:55:00Z",
|
|
||||||
"files": [
|
|
||||||
{"path": "report.md", "sha256": "sha256:...", "size": 1234}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"links": {
|
|
||||||
"primary_url": "https://reports.example.com/archive/report.html"
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"path": "report.html",
|
|
||||||
"kind": "generated",
|
|
||||||
"source_path": "report.md",
|
|
||||||
"transform": "markdown_to_html",
|
|
||||||
"url": "https://reports.example.com/archive/report.html",
|
|
||||||
"sha256": "sha256:...",
|
|
||||||
"size": 2345,
|
|
||||||
"created_at": "2026-06-04T12:00:00Z",
|
|
||||||
"updated_at": "2026-06-04T12:00:00Z"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Required fields:
|
|
||||||
|
|
||||||
- `schema_version`: must be `2` for newly written state.
|
|
||||||
- `pipeline_id`: configured pipeline id that wrote the state.
|
|
||||||
- `destination_id`: configured destination id that wrote the state.
|
|
||||||
- `published_at`: RFC3339 publication timestamp.
|
|
||||||
- `created_at`: RFC3339 timestamp for when this destination state record was first created.
|
|
||||||
- `updated_at`: RFC3339 timestamp for the latest state update.
|
|
||||||
- `state.mode`: must be `single_owner`.
|
|
||||||
- `reconciliation.mode`: `replace` or `merge`.
|
|
||||||
- `source.manifest`: embedded source bundle manifest.
|
|
||||||
- `outputs`: output records array; empty is allowed, but the field is required.
|
|
||||||
|
|
||||||
Optional fields:
|
|
||||||
|
|
||||||
- `distributor_version`: application version string when available.
|
|
||||||
- `links.primary_url`: absolute HTTP or HTTPS URL selected by destination link policy.
|
|
||||||
|
|
||||||
## Output Records
|
|
||||||
|
|
||||||
Each output record has:
|
|
||||||
|
|
||||||
- `path`: destination-relative output path.
|
|
||||||
- `kind`: `source` or `generated`.
|
|
||||||
- `source_path`: source manifest path used for the output.
|
|
||||||
- `transform`: required for `generated` outputs; omitted for copied source outputs.
|
|
||||||
- `url`: optional absolute HTTP or HTTPS URL for the output.
|
|
||||||
- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes.
|
|
||||||
- `size`: output byte size, zero or greater.
|
|
||||||
- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed.
|
|
||||||
- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state.
|
|
||||||
|
|
||||||
Output paths must be unique and use clean relative slash-separated path rules.
|
|
||||||
|
|
||||||
For replacement updates, newly planned outputs are written into state. For merge updates, retained output records preserve both timestamps, overwritten managed output records preserve `created_at` and receive a new `updated_at`, and new output records receive the current publication time for both fields.
|
|
||||||
|
|
||||||
## Reconciliation Semantics
|
|
||||||
|
|
||||||
Destination reconciliation applies when destination state is older than the source and transfer policy permits replacement.
|
|
||||||
|
|
||||||
- `replace`: for single-owner state, delete managed output paths recorded in `outputs` plus `.distributor.json`, require the destination bundle path to be empty afterward, write the newly planned outputs, and write state whose `outputs` are exactly that new planned set. For shared-root state, delete only the current owner's omitted outputs and preserve unrelated owners.
|
|
||||||
- `merge`: retain managed output paths omitted from the new plan, overwrite planned paths only when they are already recorded in existing state, fail when a newly planned path exists in storage but is not recorded as managed, and write state whose `outputs` are the cumulative managed set.
|
|
||||||
|
|
||||||
Takeover replacement applies when valid managed state has an identity, source, or shared-root output-owner mismatch and destination `takeover.mode` permits the current publication to take ownership. It uses bounded managed replacement behavior. For shared-root state, only taken-over output records and the current owner records are rewritten; unrelated owner records and non-conflicting outputs remain managed by their existing owners.
|
|
||||||
|
|
||||||
Top-level `links.primary_url` is selected from the newly planned outputs for the current publication. Retained outputs keep their existing per-output URL metadata.
|
|
||||||
|
|
||||||
If a same-source merge publication fails after writing outputs, cleanup removes only newly created outputs from that failed attempt. Previously managed overwritten paths remain managed and are not removed by failed-attempt cleanup. Takeover replacement does not retain omitted outputs through merge reconciliation.
|
|
||||||
|
|
||||||
## Comparison Semantics
|
|
||||||
|
|
||||||
`distributor` compares the current source manifest to destination state before writing:
|
|
||||||
|
|
||||||
- No state and no content: publish new outputs.
|
|
||||||
- No state and existing content: treat the destination as unmanaged.
|
|
||||||
- Shared-root state without the current owner: publish new outputs for that owner if planned paths do not collide with unmanaged content or with other owners that `takeover.mode` does not permit.
|
|
||||||
- Matching embedded source manifest: skip.
|
|
||||||
- Same source id with older `created`: replace if policy allows.
|
|
||||||
- Same source id with newer `created`: skip by default, or replace when `transfer.on_destination_newer: replace` is configured.
|
|
||||||
- Same source id and same `created` with different digest: conflict by default, or replace when `transfer.on_conflict: replace` is configured.
|
|
||||||
- Different source id, pipeline id, or destination id in single-owner state: conflict unless `takeover.mode` permits managed ownership transfer or `transfer.on_conflict: replace` is configured.
|
|
||||||
- Shared-root output path owned by a different owner: conflict unless `takeover.mode` permits managed ownership transfer or `transfer.on_conflict: replace` is configured.
|
|
||||||
- Invalid state JSON or invalid state fields: conflict.
|
|
||||||
|
|
||||||
Normal single-owner replacement deletes only managed output paths recorded in `outputs` plus `.distributor.json`. Shared-root replacement deletes only omitted outputs for the current owner. Merge publication retains omitted managed outputs for same-source replacement. Forced replacement deletes the bounded destination bundle path.
|
|
||||||
|
|
||||||
## State Repair Semantics
|
|
||||||
|
|
||||||
`distributor reconcile-state` can remove managed output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
|
||||||
|
|
||||||
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. The command checks output paths recorded in `outputs`, reports missing managed outputs, reports unmanaged entries under the destination root, and removes missing output records from valid state unless `--dry-run` is set.
|
|
||||||
|
|
||||||
For shared-root state, repair is scoped to the selected owner by default. With `--all-owners`, it checks and repairs missing output records for every owner in the selected shared-root state file.
|
|
||||||
|
|
||||||
State repair does not validate output digests, delete destination files, adopt unmanaged entries, or rewrite invalid or mismatched state.
|
|
||||||
|
|
||||||
## Prune Semantics
|
|
||||||
|
|
||||||
`distributor prune` can delete managed outputs selected by the configured destination retention policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
|
||||||
|
|
||||||
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
|
|
||||||
|
|
||||||
Prune planning uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`.
|
|
||||||
|
|
||||||
## Compatibility
|
|
||||||
|
|
||||||
`distributor` can read schema version `1` destination state for compatibility. When v1 state is read, it is treated as single-owner state with `reconciliation.mode: replace`. Missing top-level `created_at` and `updated_at` are inferred from `published_at`, and missing per-output timestamps are also inferred from `published_at`.
|
|
||||||
|
|
||||||
Newly written single-owner destination state from publish execution uses schema version `2`.
|
|
||||||
|
|
||||||
## Shared-Root State Schema
|
|
||||||
|
|
||||||
State written by `run` for `state.mode: shared_root` uses schema version `3`.
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"schema_version": 3,
|
|
||||||
"distributor_version": "dev",
|
"distributor_version": "dev",
|
||||||
"created_at": "2026-06-04T12:00:00Z",
|
"created_at": "2026-06-04T12:00:00Z",
|
||||||
"updated_at": "2026-06-04T12:10:00Z",
|
"updated_at": "2026-06-04T12:10:00Z",
|
||||||
"state": {
|
"state": {
|
||||||
"mode": "shared_root"
|
"mode": "catalog"
|
||||||
},
|
},
|
||||||
"owners": [
|
|
||||||
{
|
|
||||||
"pipeline_id": "reports",
|
|
||||||
"destination_id": "archive",
|
|
||||||
"reconciliation": {
|
|
||||||
"mode": "merge"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"manifest": {
|
|
||||||
"schema_version": 1,
|
|
||||||
"id": "reports.example.2026-06-04",
|
|
||||||
"digest": "sha256:...",
|
|
||||||
"created": "2026-06-04T11:55:00Z",
|
|
||||||
"files": [
|
|
||||||
{"path": "report.md", "sha256": "sha256:...", "size": 1234}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"links": {
|
|
||||||
"primary_url": "https://reports.example.com/archive/report.html"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"path": "report.html",
|
"path": "report.html",
|
||||||
|
"pipeline_id": "reports",
|
||||||
|
"destination_id": "static-site",
|
||||||
|
"source": {
|
||||||
|
"id": "reports.example.2026-06-04",
|
||||||
|
"digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||||
|
"created": "2026-06-04T11:55:00Z"
|
||||||
|
},
|
||||||
"kind": "generated",
|
"kind": "generated",
|
||||||
"source_path": "report.md",
|
"source_path": "report.md",
|
||||||
"transform": "markdown_to_html",
|
"transform": "markdown_to_html",
|
||||||
"url": "https://reports.example.com/archive/report.html",
|
"url": "https://reports.example.com/archive/report.html",
|
||||||
"sha256": "sha256:...",
|
"sha256": "sha256:abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd",
|
||||||
"size": 2345,
|
"size": 2345,
|
||||||
"pipeline_id": "reports",
|
|
||||||
"destination_id": "archive",
|
|
||||||
"source_id": "reports.example.2026-06-04",
|
|
||||||
"source_digest": "sha256:...",
|
|
||||||
"source_created": "2026-06-04T11:55:00Z",
|
|
||||||
"created_at": "2026-06-04T12:00:00Z",
|
"created_at": "2026-06-04T12:00:00Z",
|
||||||
"updated_at": "2026-06-04T12:10:00Z"
|
"updated_at": "2026-06-04T12:10:00Z"
|
||||||
}
|
}
|
||||||
@@ -200,55 +40,88 @@ State written by `run` for `state.mode: shared_root` uses schema version `3`.
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Shared-root required fields:
|
Top-level fields:
|
||||||
|
|
||||||
- `schema_version`: must be `3`.
|
- `schema_version`: required value `4`.
|
||||||
- `created_at`: RFC3339 timestamp for when this shared-root state record was first created.
|
- `distributor_version`: optional application version string.
|
||||||
- `updated_at`: RFC3339 timestamp for the latest shared-root state update.
|
- `created_at`: RFC3339 timestamp for when this catalog record was first created.
|
||||||
- `state.mode`: must be `shared_root`.
|
- `updated_at`: RFC3339 timestamp for the latest catalog update.
|
||||||
- `owners`: owner records keyed by `pipeline_id` and `destination_id`; each owner records its latest source manifest and reconciliation mode.
|
- `state.mode`: required value `catalog`.
|
||||||
- `outputs`: output records for every managed path under the shared destination root.
|
- `outputs`: required array of catalog output records. Empty is valid.
|
||||||
|
|
||||||
Shared-root optional fields:
|
## Output Records
|
||||||
|
|
||||||
- `distributor_version`: application version string when available.
|
Each output record has:
|
||||||
- `owners[].links.primary_url`: absolute HTTP or HTTPS URL selected by that owner's destination link policy.
|
|
||||||
|
|
||||||
Shared-root output records carry the same `path`, `kind`, `source_path`, `transform`, `url`, `sha256`, `size`, `created_at`, and `updated_at` fields as single-owner outputs. They also include the owner `pipeline_id` and `destination_id`, plus compact source identity fields `source_id`, `source_digest`, and `source_created`.
|
- `path`: destination-bundle-relative output path.
|
||||||
|
- `pipeline_id`: configured pipeline id that manages the output path.
|
||||||
|
- `destination_id`: configured destination id that manages the output path.
|
||||||
|
- `source`: compact source identity for the output.
|
||||||
|
- `kind`: `source` or `generated`.
|
||||||
|
- `source_path`: generated outputs only; source manifest path used to derive the output.
|
||||||
|
- `transform`: generated outputs only; transform id, currently `markdown_to_html`.
|
||||||
|
- `url`: optional absolute HTTP or HTTPS URL for the output.
|
||||||
|
- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes.
|
||||||
|
- `size`: output byte size, zero or greater.
|
||||||
|
- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed.
|
||||||
|
- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state.
|
||||||
|
|
||||||
## Shared-Root Ownership
|
Output paths must be unique and use clean relative slash-separated path rules. `pipeline_id` and `destination_id` must be slug-like identifiers.
|
||||||
|
|
||||||
Shared-root state is owner-scoped by `pipeline_id` and `destination_id`.
|
For copied source outputs, `source_path` and `transform` are omitted. For generated outputs, both fields are required.
|
||||||
|
|
||||||
- One output path may be owned by only one owner.
|
## Source Identity
|
||||||
- The same owner may overwrite its own managed paths.
|
|
||||||
- A different owner planning an already owned path fails as a conflict.
|
|
||||||
- A planned path that exists in storage but is not recorded in state fails as unmanaged content unless forced replacement is explicitly selected.
|
|
||||||
|
|
||||||
When an owner publishes, unrelated owner records and output records are preserved. The publishing owner's record is updated with the latest source manifest, reconciliation mode, and latest primary URL when present.
|
Each output records the source identity that produced it:
|
||||||
|
|
||||||
## Shared-Root Timestamps
|
- `source.id`: source manifest id.
|
||||||
|
- `source.digest`: source manifest digest.
|
||||||
|
- `source.created`: source manifest creation timestamp.
|
||||||
|
|
||||||
For shared-root state:
|
The full source manifest is not embedded in catalog state. The source bundle's `manifest.json` remains the producer contract, and `.distributor.json` records only the source identity needed for catalog ownership and later maintenance workflows.
|
||||||
|
|
||||||
- top-level `created_at` remains the original shared-root state creation time;
|
## Workflow Semantics
|
||||||
- top-level `updated_at` changes after a successful state write;
|
|
||||||
- output `created_at` remains stable for an existing managed path;
|
|
||||||
- output `updated_at` changes only when that path is rewritten;
|
|
||||||
- newly managed output paths receive the publication time for both output timestamps.
|
|
||||||
|
|
||||||
## Shared-Root Migration
|
Destination `workflow` is runtime configuration and is not persisted in `.distributor.json`.
|
||||||
|
|
||||||
If `state.mode: shared_root` is configured and existing state is a compatible single-owner `.distributor.json` for the same pipeline id and destination id, the next successful publish writes schema version `3` shared-root state for that owner.
|
`workflow: additive` writes planned outputs and retains unrelated catalog-managed outputs. If a planned path already has a catalog output record, the current publication replaces that record and overwrites the file. If a planned path exists in storage but is not recorded in valid catalog state, planning fails as unmanaged unless `run --force` selects `force_replace`.
|
||||||
|
|
||||||
If `state.mode: shared_root` is configured and existing single-owner state belongs to a different pipeline or destination, publish fails as a conflict. `distributor` does not implicitly convert unrelated single-owner state or take over unmanaged files during shared-root migration.
|
`workflow: replacement` writes planned outputs for the current pipeline and destination and removes omitted outputs owned by that same pipeline and destination. Outputs owned by other pipeline/destination pairs remain catalog-managed. This is normal managed replacement and does not require `--force`.
|
||||||
|
|
||||||
|
`force_replace` is an explicit per-run recovery path. It deletes only the resolved destination bundle path, then writes planned outputs and fresh catalog state. It can replace unmanaged content, planned unmanaged path collisions, invalid destination state, and unsupported future destination state after dry-run review.
|
||||||
|
|
||||||
|
## Publish Planning Outcomes
|
||||||
|
|
||||||
|
Current run reports use these destination action labels:
|
||||||
|
|
||||||
|
- `publish_new`: no valid state exists and the destination bundle path is empty.
|
||||||
|
- `upsert_additive`: valid catalog state exists and additive workflow will write the planned outputs.
|
||||||
|
- `replace_catalog`: valid catalog state exists and replacement workflow will write the planned outputs and remove omitted outputs for the current owner.
|
||||||
|
- `skip_same`: no-op action value in the run output vocabulary.
|
||||||
|
- `force_replace`: explicit bounded destructive replacement selected by `--force`.
|
||||||
|
- `fail_unmanaged`: unmanaged destination content prevents publication.
|
||||||
|
- `fail_conflict`: invalid state or unsupported state prevents publication without explicit force.
|
||||||
|
|
||||||
|
Schema versions older than `4` are superseded legacy state for publish planning. Normal catalog planning may publish over superseded legacy state according to the configured workflow, while invalid state and unsupported future schema versions fail unless `--force` is explicitly selected.
|
||||||
|
|
||||||
|
## Repair Semantics
|
||||||
|
|
||||||
|
`distributor reconcile-state` removes catalog output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||||
|
|
||||||
|
By default, repair is scoped to output records whose `pipeline_id` and `destination_id` match the selected pipeline and destination. With `--all-owners`, it checks every catalog output record in the selected root.
|
||||||
|
|
||||||
|
The command reports missing managed outputs and unmanaged storage entries. Without `--dry-run`, it removes missing managed output records from valid catalog state and rewrites `.distributor.json`. It does not delete destination files, adopt unmanaged entries, validate output digests, or rewrite invalid state.
|
||||||
|
|
||||||
|
## Prune Semantics
|
||||||
|
|
||||||
|
`distributor prune` deletes catalog output paths selected by the configured destination `retention.prune` policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||||
|
|
||||||
|
Prune planning is scoped to output records whose `pipeline_id` and `destination_id` match the selected pipeline and destination. It uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid catalog state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
Destination state is internal managed state written by `distributor`. Operators may inspect it during recovery, but normal workflows should not edit it by hand. Source `manifest.json` is not copied as destination state.
|
Destination state is internal managed state written by `distributor`. Operators may inspect it during recovery, but normal workflows should not edit it by hand. Source `manifest.json` is not copied as destination state.
|
||||||
|
|
||||||
## Tests
|
|
||||||
|
|
||||||
Before changing this contract, inspect and run:
|
Before changing this contract, inspect and run:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ Destination state stores generated outputs with `kind: generated`, `source_path`
|
|||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose transfer policy. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata.
|
Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose catalog workflow behavior. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ The adapter uses these S3 operations:
|
|||||||
|
|
||||||
Writes buffer the input and set `ContentLength`. If no content type is supplied by the caller, the adapter infers a content type from the logical path.
|
Writes buffer the input and set `ContentLength`. If no content type is supplied by the caller, the adapter infers a content type from the logical path.
|
||||||
|
|
||||||
Normal replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Same-source merge publication retains omitted managed objects and may overwrite existing managed objects. Takeover replacement does not retain omitted outputs through merge reconciliation. Forced replacement deletes objects under the bounded destination bundle prefix. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
|
Normal managed replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Forced replacement deletes objects under the bounded destination bundle prefix, then writes planned outputs and schema version `4` catalog state. For fixed-path destinations, that bounded prefix is the configured backend root. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
|
||||||
|
|
||||||
## Error Mapping
|
## Error Mapping
|
||||||
|
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ The configured `path` is the backend root. All source discovery, destination pat
|
|||||||
|
|
||||||
The adapter rejects symlink ancestors for reads and writes. Reads require regular files. Writes create parent directories and prefer atomic temp-file-plus-rename writes when requested. Walk output is sorted through the shared storage walker.
|
The adapter rejects symlink ancestors for reads and writes. Reads require regular files. Writes create parent directories and prefer atomic temp-file-plus-rename writes when requested. Walk output is sorted through the shared storage walker.
|
||||||
|
|
||||||
Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Same-source merge publication retains omitted managed paths and may overwrite existing managed paths. Takeover replacement does not retain omitted outputs through merge reconciliation. Forced replacement deletes the bounded destination bundle path.
|
Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Forced replacement deletes the bounded destination bundle path, then writes planned outputs and schema version `4` catalog state. For fixed-path destinations, that bounded path is the configured backend root.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ User-facing command parsing stays in `internal/cli`, including `reconcile-state`
|
|||||||
|
|
||||||
## Config Fields Used
|
## Config Fields Used
|
||||||
|
|
||||||
The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, takeover policy, retention policy, and transfer policy.
|
The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, workflow, publish policy, transform policy, path mapping, links, and retention policy.
|
||||||
|
|
||||||
Config fields are validated and defaulted by `internal/config` before app workflows use them.
|
Config fields are validated and defaulted by `internal/config` before app workflows use them.
|
||||||
|
|
||||||
@@ -32,11 +32,11 @@ The app layer registers default transforms, including Markdown-to-HTML, and supp
|
|||||||
|
|
||||||
## State And Manifest Behavior
|
## State And Manifest Behavior
|
||||||
|
|
||||||
Run workflows discover and validate source bundles through `internal/bundle`. Destination state actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results. Run summaries count older-state, conflict, newer-state, takeover, and explicit forced replacements separately.
|
Run workflows discover and validate source bundles through `internal/bundle`. Destination catalog actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results. Run summaries count `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and `fail_conflict` separately.
|
||||||
|
|
||||||
Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing managed output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from state and rewrites valid state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid or mismatched state.
|
Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing catalog output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from valid catalog state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid state.
|
||||||
|
|
||||||
Prune planning consumes a parsed destination state document and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state.
|
Prune planning consumes parsed catalog state and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state.
|
||||||
|
|
||||||
Prune execution loads one configured pipeline/destination selector, opens that destination root, parses the root `.distributor.json`, and builds a plan from the destination retention policy. Dry-run returns the same planned and preserved managed output records without deleting files or rewriting state. Apply mode deletes only planned managed output paths, never unmanaged files or `.distributor.json`, then removes confirmed deleted records from state and updates the state timestamp. If a delete fails after earlier deletes succeeded, it rewrites state only for the confirmed deletions and preserves records for the failed and unattempted outputs so a retry remains accurate. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope.
|
Prune execution loads one configured pipeline/destination selector, opens that destination root, parses the root `.distributor.json`, and builds a plan from the destination retention policy. Dry-run returns the same planned and preserved managed output records without deleting files or rewriting state. Apply mode deletes only planned managed output paths, never unmanaged files or `.distributor.json`, then removes confirmed deleted records from state and updates the state timestamp. If a delete fails after earlier deletes succeeded, it rewrites state only for the confirmed deletions and preserves records for the failed and unattempted outputs so a retry remains accurate. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope.
|
||||||
|
|
||||||
@@ -54,9 +54,9 @@ HTTP upload status is in memory. Accepted jobs move through accepted, queued, ru
|
|||||||
|
|
||||||
Runtime setup fails for config load, config validation, secret loading, or credential resolution errors. Source setup failures stop the affected run before destination planning. Destination open, planning, execution, and notification failures are recorded as destination failures where a partial result exists.
|
Runtime setup fails for config load, config validation, secret loading, or credential resolution errors. Source setup failures stop the affected run before destination planning. Destination open, planning, execution, and notification failures are recorded as destination failures where a partial result exists.
|
||||||
|
|
||||||
Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root all-owner repair still uses the selected destination to identify the root, then applies repair across owners inside that root. Invalid, unreadable, or ambiguous state fails before any rewrite.
|
Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Catalog repair is scoped to that owner unless `--all-owners` is set. Invalid or unreadable state fails before any rewrite.
|
||||||
|
|
||||||
Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root pruning is scoped to the selected owner and preserves unrelated owners. Invalid, unreadable, or ambiguous state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output.
|
Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Pruning is scoped to the selected owner and preserves unrelated owners. Invalid or unreadable state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output.
|
||||||
|
|
||||||
HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Upload requests can fail during authentication, idempotency-key validation, content-type validation, idempotency conflict checks, queue admission, archive staging, source validation, or later publish execution.
|
HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Upload requests can fail during authentication, idempotency-key validation, content-type validation, idempotency conflict checks, queue admission, archive staging, source validation, or later publish execution.
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ The canonical user-facing config reference is `docs/config.md`.
|
|||||||
|
|
||||||
## Config Fields Used
|
## Config Fields Used
|
||||||
|
|
||||||
The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, takeover policy, retention policy, and transfer policy.
|
The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, destination workflow, publish policy, transform policy, path mapping, links, and retention policy.
|
||||||
|
|
||||||
## Adapters Used
|
## Adapters Used
|
||||||
|
|
||||||
@@ -26,11 +26,11 @@ No external storage adapters are used directly. The package exposes normalized c
|
|||||||
|
|
||||||
## State And Manifest Behavior
|
## State And Manifest Behavior
|
||||||
|
|
||||||
The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as publish/transform combinations, links, state policy, reconciliation policy, takeover policy, retention policy, transfer policy, backend roots, S3 prefix shape, and HTTP upload source settings.
|
The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as workflow, publish/transform combinations, links, retention policy, backend roots, S3 prefix shape, and HTTP upload source settings. Legacy destination policy YAML fields are rejected by strict decoding because they are not user-visible config fields.
|
||||||
|
|
||||||
## Skip And Resume Behavior
|
## Skip And Resume Behavior
|
||||||
|
|
||||||
The package has no runtime skip or resume behavior. It provides transfer policy values that publish planning later applies to destination comparison outcomes.
|
The package has no runtime skip or resume behavior. Publish planning later applies workflow values and per-run force options to destination catalog outcomes.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
|
|||||||
@@ -4,13 +4,13 @@ Audience: developers and LLM coding agents changing `internal/publish`.
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns destination comparison mapping, output selection, URL planning, managed cleanup selection, replacement safety, and destination state projection.
|
`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns output selection, URL planning, catalog action selection, managed cleanup selection, forced replacement safety, and destination state projection.
|
||||||
|
|
||||||
## Inputs And Outputs
|
## Inputs And Outputs
|
||||||
|
|
||||||
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, state policy, reconciliation policy, takeover policy, transformer resolver, transfer policy, distributor version, and force flag.
|
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, workflow, transformer resolver, distributor version, force flag, and request time.
|
||||||
|
|
||||||
Output from planning is a `Plan` with action, reason, destination identity, selected outputs, state mode, owner scope, reconciliation mode, takeover mode, optional existing single-owner or shared-root state, optional primary URL, and force metadata. Shared-root plans also expose other-owner outputs to preserve, taken-over outputs, current-owner outputs retained by merge, current-owner outputs deleted by replace or takeover, and current-owner outputs to write. Execution writes selected source outputs, generated outputs, and `.distributor.json` for executable publish or replacement actions.
|
Output from planning is a `Plan` with action, reason, destination identity, selected outputs, workflow, owner scope, optional existing catalog state, optional superseded legacy marker, optional primary URL, catalog outputs to write, catalog outputs to retain, catalog outputs to delete, force metadata, and clear-root metadata. Execution writes selected source outputs, generated outputs, and schema version `4` `.distributor.json` for executable catalog actions.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
@@ -20,7 +20,7 @@ External destination state semantics are documented in `docs/integrations/destin
|
|||||||
|
|
||||||
## Config Fields Used
|
## Config Fields Used
|
||||||
|
|
||||||
The package consumes already-defaulted config values for destination `publish`, `transform`, `links`, `state`, `reconciliation`, `takeover`, `transfer`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
|
The package consumes already-defaulted config values for destination `workflow`, `publish`, `transform`, `links`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
|
||||||
|
|
||||||
## Adapters Used
|
## Adapters Used
|
||||||
|
|
||||||
@@ -28,27 +28,37 @@ The package depends on `internal/storage.Backend` for source and destination IO,
|
|||||||
|
|
||||||
## State And Manifest Behavior
|
## State And Manifest Behavior
|
||||||
|
|
||||||
Planning inspects destination state through `internal/state`, compares it with the source manifest, and maps comparison outcomes plus transfer and takeover policy into actions: `publish_new`, `replace_older`, `replace_conflict`, `replace_newer`, `replace_takeover`, `force_replace`, `skip_same`, `skip_destination_newer`, `fail_conflict`, or `fail_unmanaged`.
|
Planning inspects destination state through `internal/state` and maps catalog conditions into actions:
|
||||||
|
|
||||||
Single-owner destinations compare the whole destination state against the configured pipeline and destination ids. Valid managed identity and source conflicts can become `replace_takeover` when `takeover.mode` allows them. Shared-root destinations compare only the current owner scope, keyed by pipeline id and destination id. An absent shared-root owner is publishable for that owner unless a planned output collides with unmanaged storage content. Planned writes to a path owned by another shared-root owner become `replace_takeover` when `takeover.mode` allows that managed output path to move to the current owner.
|
- `publish_new`: no valid state exists and the destination bundle path is empty.
|
||||||
|
- `upsert_additive`: additive workflow writes planned outputs and retains unrelated catalog outputs.
|
||||||
|
- `replace_catalog`: replacement workflow writes planned outputs and deletes omitted outputs for the current owner.
|
||||||
|
- `skip_same`: valid catalog metadata already matches every planned output.
|
||||||
|
- `force_replace`: explicit bounded replacement selected by `Force`.
|
||||||
|
- `fail_unmanaged`: unmanaged destination content blocks publication.
|
||||||
|
- `fail_conflict`: invalid or unsupported state blocks publication.
|
||||||
|
|
||||||
Execution writes destination state after selected outputs are written. Destination state includes copied source output metadata, generated output metadata, output timestamps, embedded source manifest, reconciliation metadata, link metadata when configured, pipeline id, destination id, and publication timestamps.
|
Superseded legacy state is identified by schema number and planned through catalog output projection. A successful publish writes schema version `4` catalog state.
|
||||||
|
|
||||||
## Skip And Resume Behavior
|
Execution writes destination state after selected outputs are written. Catalog output records include owner identity, compact source identity, copied source output metadata, generated output metadata, output timestamps, and optional URL metadata.
|
||||||
|
|
||||||
`skip_same` and `skip_destination_newer` execute as no-ops. Replacement-mode single-owner updates remove managed output paths from existing state plus `.distributor.json`, verify the destination is empty, and write state whose outputs are exactly the new plan. `replace_conflict` uses managed replacement mechanics and does not retain omitted outputs through merge reconciliation. `replace_newer` follows the same managed replacement and merge-retention rules as `replace_older`. `replace_takeover` uses managed replacement mechanics and does not retain omitted outputs through merge reconciliation. Replacement-mode shared-root updates remove only current-owner omitted outputs and preserve unrelated owners. Shared-root takeover rewrites only the taken-over output records and current owner records. Merge-mode same-source updates retain omitted managed outputs, overwrite only paths already recorded as managed, reject unmanaged destination path collisions, and write cumulative output state. Failed writes trigger cleanup where practical; same-source merge cleanup removes only newly created outputs from the failed attempt.
|
## Workflow Behavior
|
||||||
|
|
||||||
Shared-root execution writes schema version `3` state. It preserves unrelated owner records and outputs, updates only the publishing owner metadata, preserves root `created_at`, and updates root `updated_at` after successful state writes. Compatible single-owner state for the same pipeline and destination is converted to shared-root state on successful publish.
|
Additive workflow computes a write set for the planned outputs and preserves catalog outputs for unplanned paths. Existing catalog records for planned paths are replaced by the current owner and source identity. When every planned output already matches catalog metadata, planning returns `skip_same` and ignores unrelated retained catalog outputs for the no-op decision.
|
||||||
|
|
||||||
Forced replacement is explicit per request and deletes the bounded destination bundle path before writing new outputs and state. Valid managed conflict and newer-state replacements are normal managed replacement actions when transfer policy allows them; they are not forced replacement actions.
|
Replacement workflow computes a write set for the planned outputs, preserves other-owner outputs, and deletes omitted outputs owned by the current pipeline and destination. It does not need `Force`. Matching planned outputs return `skip_same` only when replacement would not delete omitted outputs for the current owner.
|
||||||
|
|
||||||
|
Catalog skip comparison is metadata-only. It checks pipeline id, destination id, source id, source digest, source creation timestamp, output path, kind, digest, size, generated output source path, generated output transform, and output URL metadata. It does not read destination file bytes, and `skip_same` execution does not write outputs, rewrite `.distributor.json`, delete files, or notify.
|
||||||
|
|
||||||
|
Forced replacement is explicit per request. It deletes the bounded destination bundle path before writing planned outputs and schema version `4` catalog state. Catalog planning selects `force_replace` only when `Force` is true and normal planning would otherwise fail for a non-empty no-state destination, a planned path collision with unmanaged storage content, invalid destination state, or unsupported future destination state.
|
||||||
|
|
||||||
Retention pruning is not part of publish execution and does not run automatically after a successful publish. The app-level prune workflow uses destination state after publication to select managed outputs for deletion.
|
Retention pruning is not part of publish execution and does not run automatically after a successful publish. The app-level prune workflow uses destination state after publication to select managed outputs for deletion.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
Planning fails for incomplete requests, invalid publish/transform policy, invalid state mode, invalid reconciliation mode, output path collisions, invalid destination state, unmanaged destination content without force, shared-root owner path conflicts not allowed by `takeover.mode`, conflict outcomes not allowed by transfer policy, unresolved transforms, invalid Markdown output selection, and invalid link URL planning.
|
Planning fails for incomplete requests, invalid publish/transform policy, invalid workflow, output path collisions, unresolved transforms, invalid Markdown output selection, invalid link URL planning, invalid destination state without force, unmanaged destination content without force, and unsupported future state without force.
|
||||||
|
|
||||||
Execution fails on delete, read, transform output, unmanaged merge path collision, shared-root ownership conflict, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable publish or replacement actions.
|
Execution fails on delete, read, transform output, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable catalog publish or replacement actions.
|
||||||
|
|
||||||
## Tests To Inspect
|
## Tests To Inspect
|
||||||
|
|
||||||
@@ -62,12 +72,9 @@ Execution fails on delete, read, transform output, unmanaged merge path collisio
|
|||||||
- Planning is deterministic for the same request and destination state.
|
- Planning is deterministic for the same request and destination state.
|
||||||
- Destination bundle paths are caller-supplied and backend-root-relative.
|
- Destination bundle paths are caller-supplied and backend-root-relative.
|
||||||
- URL generation uses URL path semantics and never infers public URLs from backend config.
|
- URL generation uses URL path semantics and never infers public URLs from backend config.
|
||||||
- Replacement reconciliation deletes only managed paths recorded in existing state plus `.distributor.json` for single-owner state, and only current-owner omitted outputs for shared-root state.
|
- Additive workflow never adopts unmanaged storage content.
|
||||||
- Merge reconciliation never adopts unmanaged content.
|
- Replacement workflow deletes only catalog outputs owned by the current pipeline and destination unless force replacement clears the bounded destination bundle path.
|
||||||
- Merge state output records are cumulative for the single owner.
|
- Forced replacement deletes only within the supplied destination bundle path and then writes catalog state using the same output projection as normal catalog publish planning.
|
||||||
- Shared-root planning is owner-scoped, preserves unrelated owner outputs, and records taken-over managed output paths separately from unrelated owners.
|
|
||||||
- Shared-root execution writes owner-scoped changes without deleting unrelated owners.
|
|
||||||
- Forced replacement deletes only within the supplied destination bundle path.
|
|
||||||
- Destination state is written after selected outputs are written.
|
- Destination state is written after selected outputs are written.
|
||||||
- Transform resolution stays behind a caller-supplied interface.
|
- Transform resolution stays behind a caller-supplied interface.
|
||||||
- Unmanaged content is claimed only by explicit force.
|
- Unmanaged content is claimed only by explicit force.
|
||||||
|
|||||||
@@ -4,21 +4,21 @@ Audience: developers and LLM coding agents changing `internal/state`.
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
`internal/state` parses, validates, serializes, and compares `.distributor.json` destination state.
|
`internal/state` parses, validates, serializes, and updates `.distributor.json` destination catalog state records.
|
||||||
|
|
||||||
## Inputs And Outputs
|
## Inputs And Outputs
|
||||||
|
|
||||||
Inputs are destination state JSON, constructed state values, current source manifest, pipeline id, destination id, and whether the destination path has content without state. Outputs are validated state values, JSON bytes, comparison outcomes, and human-readable reasons.
|
Inputs are destination state JSON, constructed catalog values, owner scopes, managed output paths, timestamps, and prune policy inputs. Outputs are validated catalog values, JSON bytes, managed path lists, owner-filtered output lists, missing-output repair results, and prune candidate plans.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
The package does not inspect storage backends, mutate files, choose transfer policy, build publish outputs, generate URLs, or parse config. Publish planning consumes state comparison outcomes.
|
The package does not inspect storage backends, mutate files, choose workflow actions, build publish outputs, generate URLs, or parse config. Publish planning consumes parsed catalog state and catalog output helpers.
|
||||||
|
|
||||||
The external destination state contract is documented in `docs/integrations/destination-state.md`.
|
The external destination state contract is documented in `docs/integrations/destination-state.md`.
|
||||||
|
|
||||||
## Config Fields Used
|
## Config Fields Used
|
||||||
|
|
||||||
`internal/state` uses config state mode and reconciliation mode constants for destination state validation and legacy state normalization. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
|
`internal/state` uses shared constants for catalog mode, output kinds, slug-like id validation, link validation, storage path validation, and source manifest validation. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
|
||||||
|
|
||||||
## Adapters Used
|
## Adapters Used
|
||||||
|
|
||||||
@@ -26,42 +26,34 @@ None.
|
|||||||
|
|
||||||
## State And Manifest Behavior
|
## State And Manifest Behavior
|
||||||
|
|
||||||
`.distributor.json` schema version is `2` for newly written single-owner state. Required fields are `pipeline_id`, `destination_id`, `published_at`, `created_at`, `updated_at`, `state.mode`, `reconciliation.mode`, `source.manifest`, and `outputs`. `distributor_version` and `links` are optional.
|
Current `.distributor.json` publish output uses schema version `4` catalog state. Required top-level fields are `schema_version`, `created_at`, `updated_at`, `state.mode`, and `outputs`; `distributor_version` is optional.
|
||||||
|
|
||||||
Schema version `1` state remains readable. Parsing infers `state.mode: single_owner`, `reconciliation.mode: replace`, top-level `created_at` and `updated_at` from `published_at`, and per-output timestamps from `published_at`.
|
Each catalog output record requires a clean path, pipeline id, destination id, source identity, `source` or `generated` kind, lowercase SHA-256 digest, non-negative size, and created/updated timestamps. Generated outputs require `source_path` and `transform`; copied source outputs must omit both. Stored URLs are optional and must pass `internal/link` validation.
|
||||||
|
|
||||||
Schema version `3` is shared-root state. It records `state.mode: shared_root`, shared state timestamps, owner records keyed by pipeline id and destination id, each owner's latest source manifest and reconciliation metadata, optional owner primary links, and output records for every managed path. Shared-root output records include owner ids and compact source identity fields for source id, digest, and creation time.
|
Embedded source identity records contain source manifest id, digest, and creation timestamp. Full source manifests are not embedded in catalog state.
|
||||||
|
|
||||||
Shared-root publish conversion is explicit. Compatible single-owner state for the same pipeline and destination can be projected into the current owner scope by publish execution. Unrelated single-owner state remains a conflict.
|
The package identifies schema versions older than the current catalog schema as superseded legacy state for publish planning. It rejects invalid JSON, malformed catalog state, and unsupported future schema versions.
|
||||||
|
|
||||||
Embedded source manifests are parsed and validated through `internal/bundle`, which delegates source manifest semantics to `pkg/bundle`. Output records require clean paths, `source` or `generated` kind, valid source paths, lowercase SHA-256 digests, non-negative sizes, created and updated timestamps, and transform ids for generated outputs. Stored URLs must pass `internal/link` validation.
|
The package provides helpers for finding catalog outputs by path, filtering outputs by owner, listing managed output paths, removing missing output records for one owner or every owner, and building owner-scoped prune candidates.
|
||||||
|
|
||||||
The package also provides helpers for finding output records by path, projecting planned publish outputs into timestamped state outputs, merging retained and newly planned output records, computing managed output paths from single-owner state, removing missing managed output records from single-owner state, and building prune candidates from managed outputs.
|
Publish execution owns catalog output projection and timestamp preservation for rewritten outputs. State helpers only parse, validate, filter, and remove catalog records supplied by callers.
|
||||||
|
|
||||||
For shared-root state, helpers parse either state shape, identify the current owner scope, return an owner's latest source manifest, list managed paths for one owner or all owners, detect path ownership conflicts, project planned owner outputs, merge one owner's planned outputs while preserving unrelated owners, replace one owner's outputs by removing that owner's omitted outputs, remove missing managed output records for either the current owner or every owner, and build owner-scoped prune candidates.
|
|
||||||
|
|
||||||
Shared-root helper projections preserve output `created_at` for existing managed paths and use the current publication time for rewritten `updated_at`. Root-level `created_at` preservation is owned by publish execution.
|
|
||||||
|
|
||||||
## Skip And Resume Behavior
|
## Skip And Resume Behavior
|
||||||
|
|
||||||
Comparison is pure. It returns outcomes for absent state, unmanaged content, invalid state, pipeline/destination mismatch, same source manifest, older destination, newer destination, same-created digest conflict, and different source id conflict. Shared-root helpers expose structured output ownership conflicts. State code does not decide whether to skip, replace, take over, force, or fail; publish planning maps outcomes and conflicts to actions.
|
Catalog parsing and helper transformations are pure. State code does not decide whether to skip, upsert, replace, force, or fail; publish planning maps parsed state and storage observations to actions.
|
||||||
|
|
||||||
Shared-root comparison is owner-scoped. It compares only the owner keyed by the current pipeline id and destination id, treats an absent owner as absent destination state for that owner, and can compare compatible single-owner state for the current owner without converting unrelated single-owner state.
|
Missing-output removal helpers remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers.
|
||||||
|
|
||||||
Missing-output removal helpers are pure state transformations used by app-level state repair. They remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers.
|
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution uses missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
|
||||||
|
|
||||||
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution and the `prune` command use the missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
|
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid state mode, invalid reconciliation mode, invalid embedded manifests, duplicate owners, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transforms, invalid URLs, invalid digests, negative sizes, and shared-root outputs whose owner is not registered.
|
Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid catalog mode, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transform metadata, invalid URLs, invalid digests, and negative sizes.
|
||||||
|
|
||||||
## Tests To Inspect
|
## Tests To Inspect
|
||||||
|
|
||||||
- `internal/state/distributor_test.go`
|
- `internal/state/catalog_test.go`
|
||||||
- `internal/state/shared_root_test.go`
|
|
||||||
- `internal/state/prune_test.go`
|
- `internal/state/prune_test.go`
|
||||||
- `internal/state/compare_test.go`
|
|
||||||
- `internal/app/reconcile_state_test.go`
|
- `internal/app/reconcile_state_test.go`
|
||||||
- `internal/cli/reconcile_state_test.go`
|
- `internal/cli/reconcile_state_test.go`
|
||||||
- `internal/publish/*_test.go`
|
- `internal/publish/*_test.go`
|
||||||
@@ -69,15 +61,13 @@ Parsing rejects invalid JSON, trailing data, missing required fields, invalid ti
|
|||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
|
|
||||||
- `.distributor.json` is the destination sentinel and state record.
|
- `.distributor.json` is the destination sentinel and state record.
|
||||||
- Comparison does not mutate storage.
|
- State helpers do not inspect or mutate storage.
|
||||||
- Embedded source manifests use the source bundle contract.
|
- Source identity uses the source bundle contract.
|
||||||
- Newly written single-owner state uses schema version `2`.
|
- Newly written publish state uses schema version `4`.
|
||||||
- Schema version `1` state remains readable as replacement-mode single-owner state.
|
- Superseded legacy schema handling is limited to identifying older state for publish planning.
|
||||||
- Schema version `3` shared-root state is parsed and validated without converting unrelated single-owner state.
|
|
||||||
- Shared-root owner updates preserve unrelated owners. Publish planning removes taken-over output records before owner replacement when takeover policy allows a managed path to move owners.
|
|
||||||
- Missing-output repair helpers preserve unrelated owner records and outputs.
|
- Missing-output repair helpers preserve unrelated owner records and outputs.
|
||||||
- Prune planning uses output `updated_at` and preserves unrelated shared-root owners.
|
- Prune planning uses output `updated_at` and preserves unrelated owners.
|
||||||
- Generated outputs always record a transform id.
|
- Generated outputs always record a transform id and source path.
|
||||||
- Output records always carry created and updated timestamps after parsing.
|
- Output records always carry created and updated timestamps after parsing.
|
||||||
- Stored URLs are optional and must be absolute HTTP or HTTPS URLs when present.
|
- Stored URLs are optional and must be absolute HTTP or HTTPS URLs when present.
|
||||||
- `distributor_version` is diagnostic metadata, not a comparison key.
|
- `distributor_version` is diagnostic metadata, not a comparison key.
|
||||||
|
|||||||
@@ -56,45 +56,33 @@ Destination path mapping controls where each source bundle is published beneath
|
|||||||
|
|
||||||
Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple bundles have the same timestamp, the source-root-relative bundle path in ascending order wins.
|
Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple bundles have the same timestamp, the source-root-relative bundle path in ascending order wins.
|
||||||
|
|
||||||
Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination state record. It records the pipeline id, destination id, publication time, state mode, reconciliation mode, source manifest, copied outputs, generated outputs, output timestamps, and optional public URL metadata.
|
Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination catalog. It records catalog schema version `4`, output owners, source identity for each output, output digests and sizes, timestamps, and optional URL metadata.
|
||||||
|
|
||||||
`manifest.json` from the source bundle is not copied as destination state.
|
`manifest.json` from the source bundle is not copied as destination state.
|
||||||
|
|
||||||
## Destination State And Retry Behavior
|
## Catalog Publish Behavior
|
||||||
|
|
||||||
`distributor` compares the source manifest to destination `.distributor.json` before writing:
|
`distributor` plans from the current source bundle, destination workflow, destination storage content, and `.distributor.json`.
|
||||||
|
|
||||||
- No destination state and no destination content: publish new outputs.
|
- No valid state and no destination content: publish new outputs.
|
||||||
- Matching destination state: skip as already published.
|
- No valid state and existing destination content: fail as unmanaged unless `--force` is used.
|
||||||
- Older destination state for the same source id: replace if transfer policy allows it.
|
- Valid catalog state with `workflow: additive`: write planned outputs and retain unrelated catalog outputs.
|
||||||
- Newer destination state for the same source id: skip by default, or replace when `transfer.on_destination_newer: replace` is configured.
|
- Valid catalog state with `workflow: replacement`: write planned outputs and remove omitted outputs owned by the selected pipeline and destination.
|
||||||
- Valid managed state with an identity, source, same-created digest, or shared-root output-owner mismatch: replace when destination `takeover.mode` allows it or, for remaining valid managed conflicts, when `transfer.on_conflict: replace` is configured.
|
- Planned output path exists in storage but is not recorded in valid catalog state - fail as unmanaged unless `--force` is used.
|
||||||
- Invalid destination state, identity, source, same-created digest, or shared-root output-owner mismatches not allowed by `takeover.mode` or `transfer.on_conflict: replace`: fail by default.
|
- Invalid destination state or unsupported future state - fail as conflict unless `--force` is used.
|
||||||
- Content without `.distributor.json`: fail as unmanaged content by default.
|
- Superseded legacy state - publish through the catalog planner and write schema version `4` state on success.
|
||||||
|
|
||||||
When destination state is older or newer than the source, `transfer.on_destination_older` and `transfer.on_destination_newer` control whether publication may proceed and `reconciliation.mode` controls how managed outputs are updated.
|
`workflow: additive` is the default. It is useful for archive roots, fan-out roots that intentionally receive disjoint outputs, and roots that accumulate managed outputs over time.
|
||||||
|
|
||||||
When `transfer.on_conflict: replace` permits `replace_conflict`, the destination is valid managed state but does not match the current source, pipeline, destination, or same-created digest. `replace_conflict` rewrites the affected managed destination state and does not retain omitted outputs through merge reconciliation.
|
`workflow: replacement` is useful for stable latest-style roots where the current pipeline and destination should leave only the currently planned output set for that owner. Replacement workflow is normal managed behavior and does not require `--force`.
|
||||||
|
|
||||||
For takeover replacement, `reconciliation.mode: merge` does not retain omitted outputs from the previous source identity. The destination is rewritten as a managed replacement for the current source or shared-root owner.
|
|
||||||
|
|
||||||
`reconciliation.mode: replace` is the default. It deletes only managed output paths recorded in `.distributor.json` plus the state file, verifies the destination bundle path is empty, then writes the newly planned outputs and state. The new state `outputs` array is exactly the newly planned output set.
|
|
||||||
|
|
||||||
`reconciliation.mode: merge` retains prior managed outputs that are omitted from the new plan. It overwrites planned paths only when those paths are already recorded in existing state as managed. If a newly planned path already exists in storage but is not recorded in state, publication fails as an unmanaged path collision. The new state `outputs` array is the cumulative managed output set.
|
|
||||||
|
|
||||||
For both modes, retained or overwritten paths are identified only from `.distributor.json`; unmanaged files are not adopted.
|
|
||||||
|
|
||||||
For `state.mode: shared_root`, one destination root may contain outputs from multiple pipeline/destination owners. Comparisons, replacement, and merge retention are scoped to the current owner. Outputs owned by other owners are preserved unless a planned output path is owned by another valid owner and `takeover.mode` allows moving that path to the current owner. A planned path that exists in storage but is not recorded in state fails as unmanaged content by default.
|
|
||||||
|
|
||||||
If `state.mode: shared_root` is configured on a destination whose existing single-owner state belongs to the same pipeline and destination, the next successful publish converts that state file to shared-root schema. Existing single-owner state for a different pipeline or destination remains a conflict.
|
|
||||||
|
|
||||||
If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. In `replace` mode and takeover replacement, cleanup removes outputs written during that failed attempt. In same-source `merge` mode, cleanup removes only newly created outputs from that failed attempt; overwritten managed outputs are left in place because they previously belonged to the managed set. Operators should still inspect the destination after a failed write before retrying.
|
|
||||||
|
|
||||||
Fan-out destinations are independent. If one destination fails after planning or execution begins, later destinations are still attempted. The command exits non-zero if any destination failed.
|
Fan-out destinations are independent. If one destination fails after planning or execution begins, later destinations are still attempted. The command exits non-zero if any destination failed.
|
||||||
|
|
||||||
|
If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. Operators should still inspect the destination bundle path after a failed write before retrying.
|
||||||
|
|
||||||
## Destination State Repair
|
## Destination State Repair
|
||||||
|
|
||||||
Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the state record only; it does not restore missing files.
|
Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the catalog record only; it does not restore missing files.
|
||||||
|
|
||||||
Preview the repair first:
|
Preview the repair first:
|
||||||
|
|
||||||
@@ -115,11 +103,9 @@ go run ./cmd/distributor reconcile-state \
|
|||||||
--destination <destination-id>
|
--destination <destination-id>
|
||||||
```
|
```
|
||||||
|
|
||||||
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks each managed output path with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting.
|
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks matching catalog output paths with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting.
|
||||||
|
|
||||||
Without `--dry-run`, it removes missing managed output records from valid state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid or mismatched state.
|
Without `--dry-run`, it removes missing managed output records from valid catalog state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid state. Add `--all-owners` only when every catalog owner inside the selected root should be repaired.
|
||||||
|
|
||||||
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, repair is scoped to the selected owner by default. Add `--all-owners` only when every owner in the selected shared-root state should have missing managed output records removed.
|
|
||||||
|
|
||||||
## Managed Output Pruning
|
## Managed Output Pruning
|
||||||
|
|
||||||
@@ -147,31 +133,28 @@ go run ./cmd/distributor prune \
|
|||||||
|
|
||||||
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, and plans from the selected destination's `retention.prune` policy. It uses managed output `updated_at` timestamps. When both `keep_latest` and `older_than` are configured, it preserves the newest `keep_latest` outputs before applying the age policy.
|
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, and plans from the selected destination's `retention.prune` policy. It uses managed output `updated_at` timestamps. When both `keep_latest` and `older_than` are configured, it preserves the newest `keep_latest` outputs before applying the age policy.
|
||||||
|
|
||||||
`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate.
|
`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths for the selected pipeline/destination owner, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate.
|
||||||
|
|
||||||
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
|
|
||||||
|
|
||||||
## Dry Runs And Output Review
|
## Dry Runs And Output Review
|
||||||
|
|
||||||
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, or SSH `known_hosts` entries. For reconciliation, dry runs report the same high-level action labels as execution; inspect the configured destination's `reconciliation.mode` to determine whether `replace_older` will replace the managed set or merge into it.
|
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, notifier events, or SSH `known_hosts` entries.
|
||||||
|
|
||||||
For shared-root destinations, dry runs are owner-scoped. A `replace_older` action replaces or merges only the current owner according to `reconciliation.mode`; unrelated owners remain managed by the shared-root state. Paths owned by another owner fail as conflicts unless `takeover.mode` allows ownership transfer.
|
|
||||||
|
|
||||||
Review these action labels before publishing:
|
Review these action labels before publishing:
|
||||||
|
|
||||||
- `publish_new`: destination state is absent, or a shared-root owner is absent and planned paths are publishable.
|
- `publish_new`: destination state is absent and the destination bundle path is empty.
|
||||||
- `replace_older`: destination state is older than the source.
|
- `upsert_additive`: additive workflow will write planned outputs into valid catalog state.
|
||||||
- `replace_newer`: destination state is newer than the source and `transfer.on_destination_newer: replace` allows managed replacement.
|
- `replace_catalog`: replacement workflow will write planned outputs and remove omitted outputs for the current owner.
|
||||||
- `replace_conflict`: destination state is valid managed state with a conflict and `transfer.on_conflict: replace` allows managed replacement.
|
- `skip_same`: planned outputs already match valid catalog metadata, so publication will not write outputs, rewrite `.distributor.json`, delete files, or notify.
|
||||||
- `replace_takeover`: destination state is valid managed state and `takeover.mode` allows replacement across an identity, source, or shared-root output-owner mismatch.
|
- `force_replace`: destructive catalog replacement selected because `--force` is present for unmanaged content, a planned unmanaged path collision, invalid state, or unsupported future state.
|
||||||
- `skip_same`: destination state already matches the source.
|
- `fail_unmanaged`: unmanaged destination content prevents publication.
|
||||||
- `skip_destination_newer`: destination state is newer than the source and is skipped.
|
- `fail_conflict`: invalid or unsupported destination state prevents publication.
|
||||||
- `force_replace`: destructive replacement selected because `--force` is present and policy permits it.
|
- `error`: setup, planning, or execution failed for that destination.
|
||||||
- `error`: planning or execution failed for that destination.
|
|
||||||
|
|
||||||
Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced.
|
Text and JSON summaries count `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, `fail_conflict`, and failed destinations separately. JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document.
|
||||||
|
|
||||||
Text and JSON summaries count `publish_new`, `replace_older`, `replace_conflict`, `replace_newer`, `replace_takeover`, `force_replace`, skipped, and failed destinations separately. Takeover action records include the configured takeover mode and the conflict reason. JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document.
|
The `skip_same` optimization trusts valid catalog metadata. It compares owner identity, source identity, output path, kind, digest, size, generated output metadata, and URL metadata recorded in `.distributor.json`; it does not read destination file bytes to detect bitrot.
|
||||||
|
|
||||||
|
Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced. For fixed destinations, the resolved destination bundle path is the backend root.
|
||||||
|
|
||||||
## Forced Replacement Workflow
|
## Forced Replacement Workflow
|
||||||
|
|
||||||
@@ -182,11 +165,9 @@ go run ./cmd/distributor run --config <config-path> --dry-run --force
|
|||||||
go run ./cmd/distributor run --config <config-path> --force
|
go run ./cmd/distributor run --config <config-path> --force
|
||||||
```
|
```
|
||||||
|
|
||||||
Forced replacement can claim unmanaged non-empty destination paths and is reserved for exceptional destructive replacement. Valid managed newer state and valid managed conflict state use `replace_newer` and `replace_conflict` when the corresponding transfer policy allows replacement; they do not require `--force`.
|
Forced replacement can claim a non-empty destination path with no valid `.distributor.json`, replace planned output paths that collide with storage content not recorded in valid catalog state, and recover from invalid or unsupported future destination state. It is reserved for exceptional destructive replacement. Valid catalog-managed additive upserts and replacement workflow publishes do not require `--force`.
|
||||||
|
|
||||||
Forced replacement deletes the current destination bundle path before writing outputs and state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the backend root, so a forced replacement can clear that configured root.
|
Forced replacement deletes the current destination bundle path before writing planned outputs and schema version `4` catalog state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the configured backend root, so a forced replacement can clear that configured root. Dry-run text and JSON output report that root as `target=.` or `destination_path: "."`.
|
||||||
|
|
||||||
For shared-root destinations, forced replacement also deletes the configured destination bundle path before writing new shared-root state. This removes unrelated owners inside that destination root. Preview with `--dry-run --force` and confirm the destination path before applying.
|
|
||||||
|
|
||||||
`--force` applies only to the current invocation. There is no config field that enables forced replacement by default.
|
`--force` applies only to the current invocation. There is no config field that enables forced replacement by default.
|
||||||
|
|
||||||
@@ -226,9 +207,9 @@ curl -X POST http://127.0.0.1:8080/v1/pipelines/example-http-upload/upload \
|
|||||||
--data-binary @bundle.tar.gz
|
--data-binary @bundle.tar.gz
|
||||||
```
|
```
|
||||||
|
|
||||||
Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md) for the copyable producer implementation guide.
|
Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md).
|
||||||
|
|
||||||
The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`. Set `DISTRIBUTOR_EXAMPLE_UPLOAD_PIPELINE_ID` or pass a second argument to use another configured upload pipeline. It generates an idempotency key by default; set `DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY` when retrying the same producer run across separate process runs.
|
The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go run ./examples/upload-client
|
go run ./examples/upload-client
|
||||||
@@ -250,9 +231,9 @@ Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Co
|
|||||||
|
|
||||||
Upload admission is bounded by `server.http.queue_size`. Publication concurrency is bounded by `server.http.max_concurrency`, and the coordinator does not run two uploads for the same pipeline at the same time.
|
Upload admission is bounded by `server.http.queue_size`. Publication concurrency is bounded by `server.http.max_concurrency`, and the coordinator does not run two uploads for the same pipeline at the same time.
|
||||||
|
|
||||||
`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`. If another request with the same key is still being staged before its manifest is known, the server returns a retryable `409 Conflict`. Idempotency records are memory-only and expire with completed upload status records.
|
`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`.
|
||||||
|
|
||||||
The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size. The implementation also caps extracted file count.
|
The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size.
|
||||||
|
|
||||||
The default bind address is private loopback. Put TLS, public routing, rate limiting, and external access policy in a reverse proxy or deployment layer.
|
The default bind address is private loopback. Put TLS, public routing, rate limiting, and external access policy in a reverse proxy or deployment layer.
|
||||||
|
|
||||||
@@ -272,7 +253,7 @@ S3 execution uses the AWS SDK for Go v2. See [S3-Compatible Storage Integration]
|
|||||||
|
|
||||||
When explicit S3 credential variable names are configured, both must resolve to non-empty values through the process environment or `secrets.directory`. When omitted, the AWS SDK default credential chain is used as-is.
|
When explicit S3 credential variable names are configured, both must resolve to non-empty values through the process environment or `secrets.directory`. When omitted, the AWS SDK default credential chain is used as-is.
|
||||||
|
|
||||||
Normal single-owner replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Shared-root replacement deletes only current-owner omitted output objects and rewrites the shared state object. Same-source merge publication retains omitted managed objects and may overwrite existing managed objects. Takeover replacement does not retain omitted outputs through merge reconciliation. Forced replacement deletes objects under the bounded destination bundle prefix. Distributor does not manage bucket versioning or delete markers.
|
Normal managed replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Forced replacement deletes objects under the bounded destination bundle prefix and then writes schema version `4` catalog state. Distributor does not manage bucket versioning or delete markers.
|
||||||
|
|
||||||
## Secrets Operation
|
## Secrets Operation
|
||||||
|
|
||||||
@@ -287,11 +268,10 @@ Use these recovery boundaries:
|
|||||||
- For source validation failures, regenerate the source bundle and manifest together.
|
- For source validation failures, regenerate the source bundle and manifest together.
|
||||||
- For an empty or missing destination, rerun after fixing config or storage access.
|
- For an empty or missing destination, rerun after fixing config or storage access.
|
||||||
- For unmanaged destination content, move unrelated files aside or use a different destination path before publishing.
|
- For unmanaged destination content, move unrelated files aside or use a different destination path before publishing.
|
||||||
- For shared-root ownership conflicts, change one owner so it writes a different destination path, use a separate destination root, or configure `takeover.mode` when the current owner should take over valid managed output paths.
|
- For invalid or unsupported destination state, inspect `.distributor.json`; use `--force` only after dry-run review confirms bounded replacement is intended.
|
||||||
- For missing managed output files recorded in state, run `reconcile-state --dry-run`, then apply `reconcile-state` if the missing files should no longer be considered managed.
|
- For missing managed output files recorded in state, run `reconcile-state --dry-run`, then apply `reconcile-state` if the missing files should no longer be considered managed.
|
||||||
- For configured retention cleanup, run `prune --dry-run`, then apply `prune --apply` after reviewing the managed output list.
|
- For configured retention cleanup, run `prune --dry-run`, then apply `prune --apply` after reviewing the managed output list.
|
||||||
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`. In merge mode, retained outputs may be intentional managed outputs from the prior state.
|
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`.
|
||||||
- For state conflicts, verify the source, pipeline, destination, and existing `.distributor.json` before considering `--force`.
|
|
||||||
- For HTTP upload failures, inspect `/runs/<run-id>` while retained; after expiry or restart, rely on destination state and logs/output from the publishing run.
|
- For HTTP upload failures, inspect `/runs/<run-id>` while retained; after expiry or restart, rely on destination state and logs/output from the publishing run.
|
||||||
|
|
||||||
Do not edit `.distributor.json` during normal recovery. Treat it as the managed state record used for comparison and safe cleanup.
|
Do not edit `.distributor.json` during normal recovery. Treat it as the managed state record used for comparison and safe cleanup.
|
||||||
|
|||||||
@@ -96,18 +96,15 @@ The source manifest should remain minimal. Routing, destination selection, publi
|
|||||||
|
|
||||||
Each destination bundle path is managed by `.distributor.json`. This file is both the destination sentinel and the destination state record.
|
Each destination bundle path is managed by `.distributor.json`. This file is both the destination sentinel and the destination state record.
|
||||||
|
|
||||||
Publish execution writes single-owner or shared-root destination state according to destination `state.mode`. In shared-root state, one `.distributor.json` records multiple pipeline/destination owners and every managed output carries its owner identity.
|
Publish execution writes catalog destination state. One `.distributor.json` records all managed outputs under the destination bundle path, and each output carries its owning pipeline id and destination id.
|
||||||
|
|
||||||
Single-owner state records:
|
Catalog state records:
|
||||||
|
|
||||||
- `distributor` state schema version;
|
- `distributor` state schema version;
|
||||||
- pipeline id;
|
|
||||||
- destination id;
|
|
||||||
- publication timestamp;
|
|
||||||
- state creation and update timestamps;
|
- state creation and update timestamps;
|
||||||
- single-owner state mode;
|
- catalog state mode;
|
||||||
- reconciliation mode;
|
- owner identity for each managed output;
|
||||||
- the normalized source manifest used for publication;
|
- compact source identity for each managed output;
|
||||||
- metadata for copied source outputs;
|
- metadata for copied source outputs;
|
||||||
- metadata for generated outputs, such as HTML files;
|
- metadata for generated outputs, such as HTML files;
|
||||||
- optional URL metadata for published outputs;
|
- optional URL metadata for published outputs;
|
||||||
@@ -117,40 +114,23 @@ A representative destination state file is:
|
|||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"schema_version": 2,
|
"schema_version": 4,
|
||||||
"distributor_version": "0.1.0",
|
"distributor_version": "0.1.0",
|
||||||
"pipeline_id": "weather-daily",
|
|
||||||
"destination_id": "static-html",
|
|
||||||
"published_at": "2026-05-30T11:12:00Z",
|
|
||||||
"created_at": "2026-05-30T11:12:00Z",
|
"created_at": "2026-05-30T11:12:00Z",
|
||||||
"updated_at": "2026-05-30T11:12:00Z",
|
"updated_at": "2026-05-30T11:12:00Z",
|
||||||
"state": {
|
"state": {
|
||||||
"mode": "single_owner"
|
"mode": "catalog"
|
||||||
},
|
|
||||||
"reconciliation": {
|
|
||||||
"mode": "replace"
|
|
||||||
},
|
|
||||||
"source": {
|
|
||||||
"manifest": {
|
|
||||||
"schema_version": 1,
|
|
||||||
"id": "weather.daily.brentwood.2026-05-30",
|
|
||||||
"digest": "sha256:...",
|
|
||||||
"created": "2026-05-30T11:10:00Z",
|
|
||||||
"files": [
|
|
||||||
{
|
|
||||||
"path": "report.md",
|
|
||||||
"sha256": "sha256:...",
|
|
||||||
"size": 12345
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"links": {
|
|
||||||
"primary_url": "https://reports.example.com/weather-daily/"
|
|
||||||
},
|
},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"path": "index.html",
|
"path": "index.html",
|
||||||
|
"pipeline_id": "weather-daily",
|
||||||
|
"destination_id": "static-html",
|
||||||
|
"source": {
|
||||||
|
"id": "weather.daily.brentwood.2026-05-30",
|
||||||
|
"digest": "sha256:...",
|
||||||
|
"created": "2026-05-30T11:10:00Z"
|
||||||
|
},
|
||||||
"kind": "generated",
|
"kind": "generated",
|
||||||
"source_path": "report.md",
|
"source_path": "report.md",
|
||||||
"transform": "markdown_to_html",
|
"transform": "markdown_to_html",
|
||||||
@@ -167,13 +147,11 @@ A representative destination state file is:
|
|||||||
Destination comparison rules are based on `.distributor.json`:
|
Destination comparison rules are based on `.distributor.json`:
|
||||||
|
|
||||||
- No `.distributor.json`: publish normally only if the destination bundle path is empty.
|
- No `.distributor.json`: publish normally only if the destination bundle path is empty.
|
||||||
- Existing state embeds the same normalized source manifest: skip as already published.
|
- Existing catalog state with additive workflow: write planned outputs and retain unrelated managed outputs.
|
||||||
- Existing state has the same source id and an older source `created`: replace, subject to destructive-operation safety rules.
|
- Existing catalog state with replacement workflow: write planned outputs and remove omitted outputs for the current pipeline and destination owner.
|
||||||
- Existing state has the same source id and a newer source `created`: skip by default, or replace when `transfer.on_destination_newer: replace` is configured.
|
- Planned paths that collide with unmanaged storage content fail by default.
|
||||||
- Existing state has the same source id and same `created` but different digest: fail by default, or replace as valid managed conflict state when `transfer.on_conflict: replace` is configured.
|
- Invalid or unsupported destination state fails by default.
|
||||||
- Existing valid managed state has a different source id, pipeline id, destination id, or shared-root output owner: replace when destination `takeover.mode` permits that ownership transfer, or replace as valid managed conflict state when `transfer.on_conflict: replace` is configured; otherwise fail as a conflict.
|
- Explicit forced replacement may clear the bounded destination bundle path after dry-run review.
|
||||||
|
|
||||||
For older destination state, destination `reconciliation.mode` controls output updates. `replace` rewrites the managed output set to match the new plan. `merge` retains omitted managed outputs, overwrites only existing managed paths, and rejects planned paths that collide with unmanaged storage content.
|
|
||||||
|
|
||||||
## Publication and Transform Policy
|
## Publication and Transform Policy
|
||||||
|
|
||||||
@@ -200,7 +178,7 @@ Application logic must interact with storage through internal backend interfaces
|
|||||||
|
|
||||||
Adapters should be thin. Backend adapters should implement storage operations and translate backend-specific errors, but should not make bundle comparison, transform, routing, or replacement decisions.
|
Adapters should be thin. Backend adapters should implement storage operations and translate backend-specific errors, but should not make bundle comparison, transform, routing, or replacement decisions.
|
||||||
|
|
||||||
Remote file-transfer support should prefer native protocol implementations over shelling out, unless a later design document records a reason to differ.
|
Remote file copy support should prefer native protocol implementations over shelling out, unless a later design document records a reason to differ.
|
||||||
|
|
||||||
## Dependency Policy
|
## Dependency Policy
|
||||||
|
|
||||||
@@ -221,7 +199,7 @@ Use this current layout unless the project has a documented reason to differ:
|
|||||||
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
|
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
|
||||||
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
|
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
|
||||||
- `internal/bundle`: storage-backed source bundle discovery and validation over the public manifest contract.
|
- `internal/bundle`: storage-backed source bundle discovery and validation over the public manifest contract.
|
||||||
- `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
|
- `internal/state`: `.distributor.json` catalog parsing, validation, and output metadata.
|
||||||
- `internal/link`: shared HTTP URL validation for configured and persisted link metadata.
|
- `internal/link`: shared HTTP URL validation for configured and persisted link metadata.
|
||||||
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
|
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
|
||||||
- `internal/adapters/local`: local filesystem backend.
|
- `internal/adapters/local`: local filesystem backend.
|
||||||
@@ -229,7 +207,7 @@ Use this current layout unless the project has a documented reason to differ:
|
|||||||
- `internal/adapters/s3`: S3-compatible object storage backend.
|
- `internal/adapters/s3`: S3-compatible object storage backend.
|
||||||
- `internal/transform`: transform interfaces, registry, planning, and shared transform models.
|
- `internal/transform`: transform interfaces, registry, planning, and shared transform models.
|
||||||
- `internal/transform/markdown`: Markdown-to-HTML implementation.
|
- `internal/transform/markdown`: Markdown-to-HTML implementation.
|
||||||
- `internal/publish`: destination planning, reconciliation, safety checks, and publish execution.
|
- `internal/publish`: destination planning, catalog workflow safety checks, and publish execution.
|
||||||
- `internal/notify`: notification interface and MVP no-op notifier.
|
- `internal/notify`: notification interface and MVP no-op notifier.
|
||||||
- `internal/logging`: logging setup and shared logging helpers.
|
- `internal/logging`: logging setup and shared logging helpers.
|
||||||
|
|
||||||
@@ -264,8 +242,7 @@ Pipeline configuration should express:
|
|||||||
- per-destination transform policy;
|
- per-destination transform policy;
|
||||||
- per-destination public link policy;
|
- per-destination public link policy;
|
||||||
- validation behavior;
|
- validation behavior;
|
||||||
- per-destination takeover behavior;
|
- per-destination workflow and retention behavior.
|
||||||
- destination conflict/replacement behavior.
|
|
||||||
|
|
||||||
## Modules and Registries
|
## Modules and Registries
|
||||||
|
|
||||||
@@ -295,7 +272,7 @@ Errors should be actionable and preserve context. Wrap errors with operation, pi
|
|||||||
|
|
||||||
Errors and logs must not expose secrets.
|
Errors and logs must not expose secrets.
|
||||||
|
|
||||||
Use structured logging where practical. Logs should describe discovery, validation, planned actions, skipped transfers, conflicts, replacements, external calls, retries, and failure causes, but should not include large report contents by default.
|
Use structured logging where practical. Logs should describe discovery, validation, planned actions, skipped copies, conflicts, replacements, external calls, retries, and failure causes, but should not include large report contents by default.
|
||||||
|
|
||||||
Skip and no-op decisions should be logged at an appropriate level so operators can distinguish successful publication from intentional no-op behavior.
|
Skip and no-op decisions should be logged at an appropriate level so operators can distinguish successful publication from intentional no-op behavior.
|
||||||
|
|
||||||
|
|||||||
@@ -12,13 +12,13 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
|
|||||||
- `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
|
- `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
|
||||||
- `internal/config`: YAML configuration structs, loading, defaults, and validation.
|
- `internal/config`: YAML configuration structs, loading, defaults, and validation.
|
||||||
- `internal/bundle`: storage-backed source bundle discovery and validation using the public manifest contract.
|
- `internal/bundle`: storage-backed source bundle discovery and validation using the public manifest contract.
|
||||||
- `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
|
- `internal/state`: destination `.distributor.json` catalog parsing, validation, and output metadata.
|
||||||
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
|
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
|
||||||
- `internal/adapters/local`: local filesystem backend.
|
- `internal/adapters/local`: local filesystem backend.
|
||||||
- `internal/adapters/ssh`: SSH/SFTP backend.
|
- `internal/adapters/ssh`: SSH/SFTP backend.
|
||||||
- `internal/adapters/s3`: S3-compatible object storage backend.
|
- `internal/adapters/s3`: S3-compatible object storage backend.
|
||||||
- `internal/storage/fake`: in-memory backend for tests.
|
- `internal/storage/fake`: in-memory backend for tests.
|
||||||
- `internal/publish`: destination inspection, output planning, takeover planning, reconciliation, execution, managed cleanup, and explicit forced replacement.
|
- `internal/publish`: destination inspection, output planning, catalog workflow planning, execution, managed cleanup, and explicit forced replacement.
|
||||||
- `internal/transform`: transform interface and registry.
|
- `internal/transform`: transform interface and registry.
|
||||||
- `internal/transform/markdown`: Markdown-to-HTML transform.
|
- `internal/transform/markdown`: Markdown-to-HTML transform.
|
||||||
- `internal/notify`: notification interface and current no-op notifier.
|
- `internal/notify`: notification interface and current no-op notifier.
|
||||||
|
|||||||
@@ -1,283 +0,0 @@
|
|||||||
# Catalog State And Destination Workflow Roadmap
|
|
||||||
|
|
||||||
This roadmap records the intended first-class state model and destination
|
|
||||||
workflow configuration for replacement and additive publication workflows.
|
|
||||||
|
|
||||||
Current `single_owner` and `shared_root` state are oriented around comparing a
|
|
||||||
destination owner to one latest source manifest. That works well for replacement
|
|
||||||
workflows, where a producer maintains a curated source of truth and
|
|
||||||
`distributor` makes a destination match it. It is a poor fit for additive
|
|
||||||
workflows, where each run contributes new or updated output paths while
|
|
||||||
preserving unrelated managed outputs from previous runs and other pipelines.
|
|
||||||
|
|
||||||
The target model is a current-state catalog: `.distributor.json` records the
|
|
||||||
currently managed output paths at a destination root, and each output record
|
|
||||||
stores its current owner, compact source identity, content digest, and update
|
|
||||||
metadata. It is not intended to be an audit log.
|
|
||||||
|
|
||||||
Both replacement and additive workflows should use the same catalog state shape.
|
|
||||||
The workflow choice is runtime policy from destination config, not persisted
|
|
||||||
state.
|
|
||||||
|
|
||||||
## Locked Decisions
|
|
||||||
|
|
||||||
- Add one destination state mode named `catalog` for both replacement and
|
|
||||||
additive workflows.
|
|
||||||
- Catalog state writes `.distributor.json` schema version `4`.
|
|
||||||
- Add a destination-level `workflow` setting with accepted values
|
|
||||||
`replacement` and `additive`.
|
|
||||||
- Default `workflow` to `additive` because it is the least destructive workflow.
|
|
||||||
- `workflow` replaces the normal user-facing need to combine `state.mode`,
|
|
||||||
`reconciliation.mode`, `takeover.mode`, and transfer conflict settings for the
|
|
||||||
two primary workflows.
|
|
||||||
- `workflow: replacement` treats the current planned outputs as the authoritative
|
|
||||||
managed output set for the destination scope and removes previously managed
|
|
||||||
outputs in that scope when they are no longer planned.
|
|
||||||
- `workflow: additive` writes or overwrites the planned output paths and retains
|
|
||||||
unrelated managed outputs.
|
|
||||||
- If a planned path already exists as a managed output, the new publication may
|
|
||||||
overwrite it and becomes that path's current owner.
|
|
||||||
- If a planned path exists in storage but is not recorded in valid catalog state,
|
|
||||||
it remains unmanaged content and must not be adopted implicitly.
|
|
||||||
- If an output path changes owner, preserve the output record's existing
|
|
||||||
`created_at` and update only `updated_at`.
|
|
||||||
- Catalog state records current ownership only. It does not retain historical
|
|
||||||
owners, historical sources, or old versions of overwritten output records.
|
|
||||||
- `pipeline_id` and `destination_id` are stored separately on each output
|
|
||||||
record. They are not concatenated into one owner string.
|
|
||||||
- Catalog state does not include top-level `owners`; owners are derivable from
|
|
||||||
the output records.
|
|
||||||
- Catalog state does not include top-level `sources`; compact source identity is
|
|
||||||
stored directly on each output record.
|
|
||||||
- Catalog state does not record whether the last run used `replacement` or
|
|
||||||
`additive`. Workflow is execution policy, and persisting it would create
|
|
||||||
drift risk if config changes later.
|
|
||||||
- Legacy destination policy fields should be rejected outright in the new
|
|
||||||
workflow config model. Do not retain aliases for `state`, `reconciliation`,
|
|
||||||
`takeover`, or `transfer`.
|
|
||||||
|
|
||||||
## Destination Workflow Semantics
|
|
||||||
|
|
||||||
The user-facing destination config should express intent directly:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
destinations:
|
|
||||||
- id: weather-latest
|
|
||||||
backend: local
|
|
||||||
path: /srv/reports/weather/latest
|
|
||||||
workflow: additive
|
|
||||||
```
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
destinations:
|
|
||||||
- id: weather-archive
|
|
||||||
backend: local
|
|
||||||
path: /srv/reports/weather/archive
|
|
||||||
workflow: replacement
|
|
||||||
```
|
|
||||||
|
|
||||||
`workflow` is orthogonal to backend config, path mapping, publish source/HTML
|
|
||||||
selection, transforms, links, and retention.
|
|
||||||
|
|
||||||
### Replacement Workflow
|
|
||||||
|
|
||||||
A replacement destination is for producers that maintain a curated source of
|
|
||||||
truth and expect `distributor` to make the destination's managed scope match the
|
|
||||||
current publication.
|
|
||||||
|
|
||||||
For a destination configured with `workflow: replacement`:
|
|
||||||
|
|
||||||
- planned outputs are written to their resolved destination paths;
|
|
||||||
- planned outputs may overwrite existing managed outputs at the same path;
|
|
||||||
- each written output record is replaced in-place with the current
|
|
||||||
publication's owner, source identity, hash, size, and timestamps;
|
|
||||||
- managed outputs in the destination scope that are not present in the current
|
|
||||||
plan are deleted and removed from catalog state;
|
|
||||||
- unmanaged destination content remains unmanaged and blocks planned path
|
|
||||||
collisions unless an explicit force workflow later chooses otherwise;
|
|
||||||
- the catalog state shape remains the same as additive workflow state.
|
|
||||||
|
|
||||||
### Additive Workflow
|
|
||||||
|
|
||||||
An additive destination is for producers that routinely contribute outputs to a
|
|
||||||
shared destination root.
|
|
||||||
|
|
||||||
For a destination configured with `workflow: additive`:
|
|
||||||
|
|
||||||
- planned outputs are written to their resolved destination paths;
|
|
||||||
- planned outputs may overwrite existing managed outputs at the same path;
|
|
||||||
- each overwritten output record is replaced in-place with the current
|
|
||||||
publication's owner, source identity, hash, size, and timestamps;
|
|
||||||
- managed outputs not present in the current plan are retained;
|
|
||||||
- unrelated managed outputs from other pipelines or destinations are retained;
|
|
||||||
- unmanaged destination content remains unmanaged and blocks planned path
|
|
||||||
collisions unless an explicit force workflow later chooses otherwise;
|
|
||||||
- pruning can select retained managed outputs by `updated_at` and, when useful,
|
|
||||||
by `pipeline_id` and/or `destination_id`.
|
|
||||||
|
|
||||||
Replacement workflow treats the current publication as the desired managed
|
|
||||||
output set for a destination scope. Additive workflow treats the current
|
|
||||||
publication as a patch to the catalog of currently managed outputs.
|
|
||||||
|
|
||||||
## Destination State Schema Version 4
|
|
||||||
|
|
||||||
Catalog state should use this top-level shape:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"schema_version": 4,
|
|
||||||
"distributor_version": "dev",
|
|
||||||
"created_at": "2026-06-19T12:00:00Z",
|
|
||||||
"updated_at": "2026-06-19T12:05:00Z",
|
|
||||||
"state": {
|
|
||||||
"mode": "catalog"
|
|
||||||
},
|
|
||||||
"outputs": []
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Top-level fields:
|
|
||||||
|
|
||||||
- `schema_version`: required. Value `4` for catalog state.
|
|
||||||
- `distributor_version`: optional diagnostic version string.
|
|
||||||
- `created_at`: required RFC3339 timestamp for when the catalog state file was
|
|
||||||
first created.
|
|
||||||
- `updated_at`: required RFC3339 timestamp for the latest catalog state update.
|
|
||||||
- `state.mode`: required. Value `catalog`.
|
|
||||||
- `outputs`: required array of currently managed output records.
|
|
||||||
|
|
||||||
Catalog state should not include top-level `pipeline_id`, `destination_id`,
|
|
||||||
`published_at`, `workflow`, `owners`, `sources`, or a full source manifest.
|
|
||||||
Those concepts belong on output records or in configuration.
|
|
||||||
|
|
||||||
## Output Record Schema
|
|
||||||
|
|
||||||
Each output record should be self-contained enough to support current
|
|
||||||
ownership, pruning, repair, bitrot checks, and basic provenance without a
|
|
||||||
separate owner or source catalog.
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"path": "tomorrow/index.html",
|
|
||||||
"pipeline_id": "weatherreporter.daily",
|
|
||||||
"destination_id": "latest-html",
|
|
||||||
"source": {
|
|
||||||
"id": "weatherreporter.tomorrow",
|
|
||||||
"digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
|
||||||
"created": "2026-06-19T12:00:00Z"
|
|
||||||
},
|
|
||||||
"kind": "generated",
|
|
||||||
"source_path": "report.md",
|
|
||||||
"transform": "markdown_to_html",
|
|
||||||
"sha256": "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
|
|
||||||
"size": 12345,
|
|
||||||
"created_at": "2026-06-19T12:00:00Z",
|
|
||||||
"updated_at": "2026-06-19T12:05:00Z",
|
|
||||||
"url": "https://reports.example.com/weather/tomorrow/"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Required output fields:
|
|
||||||
|
|
||||||
- `path`: destination-relative managed output path.
|
|
||||||
- `pipeline_id`: pipeline that most recently wrote this output path.
|
|
||||||
- `destination_id`: destination that most recently wrote this output path.
|
|
||||||
- `source`: compact identity of the source bundle that produced this output.
|
|
||||||
- `source.id`: source manifest id.
|
|
||||||
- `source.digest`: source manifest digest.
|
|
||||||
- `source.created`: source manifest creation timestamp, RFC3339.
|
|
||||||
- `kind`: `source` or `generated`.
|
|
||||||
- `sha256`: digest of the output bytes.
|
|
||||||
- `size`: output byte size.
|
|
||||||
- `created_at`: RFC3339 timestamp for when this output path first became
|
|
||||||
managed in catalog state.
|
|
||||||
- `updated_at`: RFC3339 timestamp for when this output path was most recently
|
|
||||||
written or updated.
|
|
||||||
|
|
||||||
Optional output fields:
|
|
||||||
|
|
||||||
- `source_path`: present only for generated outputs; source manifest path used
|
|
||||||
to produce the generated output.
|
|
||||||
- `transform`: present only for generated outputs; transform name used to
|
|
||||||
produce the generated output.
|
|
||||||
- `url`: present only when destination link configuration produces a public URL
|
|
||||||
for this output.
|
|
||||||
|
|
||||||
For copied source outputs, `source_path` and `transform` should be omitted. The
|
|
||||||
output `path` is already the copied source artifact path.
|
|
||||||
|
|
||||||
## Useful Properties
|
|
||||||
|
|
||||||
The catalog shape supports both primary workflows without unnecessary
|
|
||||||
normalization:
|
|
||||||
|
|
||||||
- current owner of each path is explicit;
|
|
||||||
- current source identity for each path is explicit;
|
|
||||||
- bitrot checks can compare storage bytes to `sha256`;
|
|
||||||
- pruning can use `updated_at`, optionally scoped by `pipeline_id` and
|
|
||||||
`destination_id`;
|
|
||||||
- repair can remove missing managed output records without consulting an owner
|
|
||||||
or source table;
|
|
||||||
- overwriting an output path updates one output record in place;
|
|
||||||
- no orphaned top-level owner/source records need to be maintained.
|
|
||||||
|
|
||||||
The schema intentionally avoids storing a full source manifest for every output.
|
|
||||||
The source manifest remains the producer-to-distributor validation contract, but
|
|
||||||
catalog state only needs compact source identity for currently managed outputs.
|
|
||||||
|
|
||||||
## Relationship To Existing State And Config Modes
|
|
||||||
|
|
||||||
Existing state modes remain current behavior until catalog mode is implemented:
|
|
||||||
|
|
||||||
- `single_owner` schema version `2` supports one owner for a destination bundle
|
|
||||||
path.
|
|
||||||
- `shared_root` schema version `3` supports multiple owners in one destination
|
|
||||||
root but still keeps owner records with latest source manifests.
|
|
||||||
- `catalog` schema version `4` should support replacement and additive
|
|
||||||
current-state ownership without top-level owner or source catalogs.
|
|
||||||
|
|
||||||
The intended user-facing config should move toward `workflow: replacement` and
|
|
||||||
`workflow: additive` instead of requiring ordinary users to combine
|
|
||||||
`state.mode`, `reconciliation.mode`, `takeover.mode`, and transfer conflict
|
|
||||||
settings.
|
|
||||||
|
|
||||||
Because the project is still alpha pre-release, catalog implementation should be
|
|
||||||
a clean break:
|
|
||||||
|
|
||||||
- remove legacy write paths for current `single_owner` and `shared_root` state;
|
|
||||||
- do not implement migration from schema versions `1`, `2`, or `3`;
|
|
||||||
- do not preserve backwards compatibility for legacy destination state;
|
|
||||||
- when a configured destination writes successfully, write schema version `4`
|
|
||||||
catalog state;
|
|
||||||
- if an existing `.distributor.json` has schema version lower than `4`, treat it
|
|
||||||
as superseded legacy state for planning purposes and overwrite it according to
|
|
||||||
the configured workflow, without attempting conversion.
|
|
||||||
|
|
||||||
For the first catalog run against superseded legacy state:
|
|
||||||
|
|
||||||
- `workflow: replacement` may clear the bounded destination root before writing
|
|
||||||
the planned outputs and schema version `4` catalog state.
|
|
||||||
- `workflow: additive` may overwrite only the planned output paths, then write
|
|
||||||
schema version `4` catalog state containing those planned outputs.
|
|
||||||
- unplanned files left behind by an additive run against superseded legacy state
|
|
||||||
are not recorded in catalog state and are treated as unmanaged content by
|
|
||||||
later catalog runs.
|
|
||||||
|
|
||||||
The target direction is that both primary workflows use catalog state.
|
|
||||||
|
|
||||||
## Prune And Reconcile-State Scope
|
|
||||||
|
|
||||||
Catalog maintenance commands should initially keep the existing ownership
|
|
||||||
selector model:
|
|
||||||
|
|
||||||
- `prune --pipeline <id> --destination <id>` operates only on outputs currently
|
|
||||||
owned by that pipeline/destination.
|
|
||||||
- `reconcile-state --pipeline <id> --destination <id>` repairs only outputs
|
|
||||||
currently owned by that pipeline/destination.
|
|
||||||
- `reconcile-state --all-owners` repairs all output records in the catalog.
|
|
||||||
- `prune` remains scoped to the selected pipeline/destination owner only in the
|
|
||||||
initial catalog implementation.
|
|
||||||
|
|
||||||
Do not add catalog-specific selectors in the initial implementation, such as
|
|
||||||
`--source-id`, `--path-prefix`, or `--kind`. Those may be useful later, but they
|
|
||||||
are not required to make additive workflow first-class.
|
|
||||||
@@ -93,10 +93,10 @@ internal docs.
|
|||||||
- Current upload status, queue, and idempotency state are memory-only.
|
- Current upload status, queue, and idempotency state are memory-only.
|
||||||
- Producers submit complete tar or gzip-compressed tar source bundles today.
|
- Producers submit complete tar or gzip-compressed tar source bundles today.
|
||||||
- Producers do not choose destination ids, destination paths, transforms, links,
|
- Producers do not choose destination ids, destination paths, transforms, links,
|
||||||
publish policy, transfer policy, storage backends, reconciliation mode, state
|
publish policy, destination workflow, storage backends, or retention policy
|
||||||
mode, or retention policy through upload requests.
|
through upload requests.
|
||||||
- Source manifests remain free of routing, destination, transform, credential,
|
- Source manifests remain free of routing, destination, transform, credential,
|
||||||
reconciliation, state, and retention data.
|
workflow, state, and retention data.
|
||||||
- Public access policy, TLS termination, and rate limiting belong outside
|
- Public access policy, TLS termination, and rate limiting belong outside
|
||||||
`distributor` unless a future implementation changes that boundary.
|
`distributor` unless a future implementation changes that boundary.
|
||||||
- `distributor` is not a broad storage synchronization tool unless a future
|
- `distributor` is not a broad storage synchronization tool unless a future
|
||||||
|
|||||||
@@ -1,399 +0,0 @@
|
|||||||
# Catalog Follow-Up Implementation Roadmap
|
|
||||||
|
|
||||||
This is the active staged plan for closing the remaining catalog-state
|
|
||||||
implementation gaps found after the schema version `4` catalog workflow landed.
|
|
||||||
It supersedes the earlier catalog implementation plan.
|
|
||||||
|
|
||||||
The current code already supports destination `workflow: additive` and
|
|
||||||
`workflow: replacement`, writes catalog state schema version `4`, rejects legacy
|
|
||||||
destination policy YAML fields, and routes publish execution through catalog
|
|
||||||
planning for normal runs. The remaining work is to make the implemented behavior
|
|
||||||
release-ready: restore a meaningful catalog `--force` path, synchronize current
|
|
||||||
docs and examples, and remove legacy single-owner/shared-root implementation
|
|
||||||
debt.
|
|
||||||
|
|
||||||
## Current Baseline
|
|
||||||
|
|
||||||
- Destination config accepts `workflow: additive` and `workflow: replacement`.
|
|
||||||
- Omitted `workflow` defaults to `additive`.
|
|
||||||
- Destination YAML fields `state`, `reconciliation`, `takeover`, and `transfer`
|
|
||||||
are rejected by strict YAML decoding.
|
|
||||||
- Successful publish writes `.distributor.json` schema version `4` with
|
|
||||||
`state.mode: catalog`.
|
|
||||||
- Schema versions `1`, `2`, and `3` are treated as superseded legacy state for
|
|
||||||
publish planning.
|
|
||||||
- `run --force` remains in the CLI and docs, but catalog publish planning does
|
|
||||||
not currently select `force_replace`.
|
|
||||||
- Current docs and examples still describe legacy config/state behavior in
|
|
||||||
several places.
|
|
||||||
- Legacy single-owner and shared-root state types, tests, and execution helpers
|
|
||||||
remain in the codebase even though normal publish planning no longer uses
|
|
||||||
them.
|
|
||||||
|
|
||||||
## Implementation Principles
|
|
||||||
|
|
||||||
- Preserve the catalog state shape defined in `docs/roadmap/catalog.md`.
|
|
||||||
- Keep `workflow` as runtime config policy; do not persist it in
|
|
||||||
`.distributor.json`.
|
|
||||||
- Keep unmanaged content protected by default.
|
|
||||||
- Keep `--force` explicit, per-run, dry-runnable, and bounded to the configured
|
|
||||||
destination bundle path.
|
|
||||||
- Do not reintroduce legacy config aliases or compatibility migration.
|
|
||||||
- Keep backend adapters unaware of catalog, workflow, and force policy.
|
|
||||||
- Update current-behavior docs in the same stage that makes the described
|
|
||||||
behavior true.
|
|
||||||
|
|
||||||
## Active Implementation Stages
|
|
||||||
|
|
||||||
## Stage 1: Catalog Force Planning And Execution
|
|
||||||
|
|
||||||
Goal: make `run --force` meaningful for catalog workflows while keeping normal
|
|
||||||
catalog safety conservative.
|
|
||||||
|
|
||||||
Implementation scope:
|
|
||||||
|
|
||||||
- Update `internal/publish` planning so `req.Force` can select
|
|
||||||
`force_replace` for exceptional destructive catalog cases.
|
|
||||||
- Force should apply to these cases:
|
|
||||||
- no valid `.distributor.json` and destination bundle path has unmanaged
|
|
||||||
content;
|
|
||||||
- planned path collision with storage content not recorded in valid catalog
|
|
||||||
state;
|
|
||||||
- invalid destination state JSON or invalid destination state fields;
|
|
||||||
- unsupported future destination state schema, if the operator explicitly
|
|
||||||
chooses force replacement after dry-run review.
|
|
||||||
- Force should not be needed for ordinary valid catalog-managed upserts or
|
|
||||||
replacements. Additive and replacement workflow behavior remains normal managed
|
|
||||||
behavior.
|
|
||||||
- `force_replace` must delete only the bounded destination bundle path through
|
|
||||||
the storage abstraction, then write planned outputs and schema version `4`
|
|
||||||
catalog state.
|
|
||||||
- For fixed-path destinations, the bounded destination bundle path is the
|
|
||||||
configured backend root. Dry-run output must make that clear.
|
|
||||||
- Preserve default behavior without `--force`:
|
|
||||||
- unmanaged planned path collisions fail as `fail_unmanaged`;
|
|
||||||
- invalid or future state fails as `fail_conflict`;
|
|
||||||
- no-state non-empty destinations fail as unmanaged.
|
|
||||||
- Ensure `force_replace` state creation uses the same catalog output projection
|
|
||||||
as normal publish planning.
|
|
||||||
- Keep `Force` out of config. There must be no persistent force default.
|
|
||||||
- Remove or update skipped force tests that were disabled during catalog
|
|
||||||
implementation.
|
|
||||||
|
|
||||||
Current-behavior documentation updates:
|
|
||||||
|
|
||||||
- Update only the force-related sections of:
|
|
||||||
- `docs/cli.md`;
|
|
||||||
- `docs/operations.md`;
|
|
||||||
- `docs/troubleshooting.md`;
|
|
||||||
- `docs/internal/publish.md`;
|
|
||||||
- backend integration docs where they describe forced deletion boundaries.
|
|
||||||
- Document force as an exceptional catalog recovery/replacement workflow, not as
|
|
||||||
a normal way to handle valid managed catalog state.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- `go test ./internal/publish`
|
|
||||||
- `go test ./internal/app ./internal/cli`
|
|
||||||
- `go test ./internal/adapters/local ./internal/storage ./internal/storage/fake`
|
|
||||||
- Planning tests:
|
|
||||||
- no-state non-empty destination returns `fail_unmanaged` without force and
|
|
||||||
`force_replace` with force;
|
|
||||||
- unmanaged planned path collision returns `fail_unmanaged` without force and
|
|
||||||
`force_replace` with force;
|
|
||||||
- invalid state returns `fail_conflict` without force and `force_replace` with
|
|
||||||
force;
|
|
||||||
- future schema returns `fail_conflict` without force and `force_replace` with
|
|
||||||
force;
|
|
||||||
- valid catalog additive and replacement actions do not become
|
|
||||||
`force_replace`.
|
|
||||||
- Execution tests:
|
|
||||||
- `force_replace` clears only the destination bundle path;
|
|
||||||
- sibling paths outside the bundle path survive for local, fake S3, and fake
|
|
||||||
SSH-style backends;
|
|
||||||
- fixed-path force clears the configured backend root and is reported as such;
|
|
||||||
- resulting state is schema version `4` catalog state.
|
|
||||||
- CLI/app tests:
|
|
||||||
- dry-run `--force` reports `force_replace` without writing;
|
|
||||||
- normal `--force` executes and increments only the `force_replace` counter;
|
|
||||||
- JSON output includes stable `force_replace` action and summary fields.
|
|
||||||
|
|
||||||
Completion criteria:
|
|
||||||
|
|
||||||
- `run --force` has observable, tested catalog behavior.
|
|
||||||
- Force is still unnecessary for normal managed catalog replacement/upsert.
|
|
||||||
- Unmanaged/invalid/future-state replacement remains impossible without explicit
|
|
||||||
`--force`.
|
|
||||||
|
|
||||||
## Stage 2: Current Documentation And Example Synchronization
|
|
||||||
|
|
||||||
Goal: make all current-behavior docs and copyable examples match catalog schema
|
|
||||||
version `4` and the `workflow` config model.
|
|
||||||
|
|
||||||
Implementation scope:
|
|
||||||
|
|
||||||
- Rewrite `docs/config.md` so destination behavior is described through:
|
|
||||||
- `workflow: additive`;
|
|
||||||
- `workflow: replacement`;
|
|
||||||
- `publish`;
|
|
||||||
- `transform`;
|
|
||||||
- `path_mapping`;
|
|
||||||
- `links`;
|
|
||||||
- `retention`.
|
|
||||||
- Remove current-behavior documentation for destination config fields:
|
|
||||||
- `state`;
|
|
||||||
- `reconciliation`;
|
|
||||||
- `takeover`;
|
|
||||||
- `transfer`.
|
|
||||||
- Rewrite `docs/integrations/destination-state.md` around schema version `4`
|
|
||||||
catalog state:
|
|
||||||
- top-level catalog fields;
|
|
||||||
- output record fields;
|
|
||||||
- source identity fields;
|
|
||||||
- generated-output-only `source_path` and `transform`;
|
|
||||||
- optional per-output `url`;
|
|
||||||
- superseded legacy schema handling for publish planning.
|
|
||||||
- Update `docs/operations.md`:
|
|
||||||
- additive workflow semantics;
|
|
||||||
- replacement workflow semantics;
|
|
||||||
- catalog `force_replace`;
|
|
||||||
- prune and reconcile-state catalog behavior;
|
|
||||||
- dry-run action labels.
|
|
||||||
- Update `docs/troubleshooting.md`:
|
|
||||||
- replace legacy conflict guidance with workflow/catalog guidance;
|
|
||||||
- describe rejected legacy config fields;
|
|
||||||
- describe force-only exceptional cases.
|
|
||||||
- Update `docs/cli.md`:
|
|
||||||
- run summary counters now use `publish_new`, `upsert_additive`,
|
|
||||||
`replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and
|
|
||||||
`fail_conflict`;
|
|
||||||
- remove legacy action labels from current command reference.
|
|
||||||
- Update relevant internal docs:
|
|
||||||
- `docs/internal/app.md`;
|
|
||||||
- `docs/internal/config.md`;
|
|
||||||
- `docs/internal/publish.md`;
|
|
||||||
- `docs/internal/state.md`.
|
|
||||||
- Update policy docs where current architectural text still describes
|
|
||||||
single-owner/shared-root behavior as current:
|
|
||||||
- `docs/policy/architecture.md`;
|
|
||||||
- `docs/policy/development.md`.
|
|
||||||
- Update examples:
|
|
||||||
- remove or rewrite `examples/shared-root.yml`;
|
|
||||||
- remove or rewrite `examples/merge-reconciliation.yml`;
|
|
||||||
- ensure examples use `workflow` where workflow intent matters;
|
|
||||||
- keep examples valid, secret-free, and copyable.
|
|
||||||
- Do not document any future catalog selectors, state migration, or compatibility
|
|
||||||
aliases outside `docs/roadmap/`.
|
|
||||||
|
|
||||||
Tests and checks:
|
|
||||||
|
|
||||||
- `go test ./internal/config ./internal/cli`
|
|
||||||
- `go test ./...`
|
|
||||||
- Run the config/example checks already used by the test suite.
|
|
||||||
- Manual smoke checks:
|
|
||||||
- `go run ./cmd/distributor run --config examples/local-publish.yml --dry-run`
|
|
||||||
- `go run ./cmd/distributor run --config examples/archive-and-latest.yml --dry-run`
|
|
||||||
- smoke any rewritten replacement/additive examples.
|
|
||||||
- Consistency searches:
|
|
||||||
- `rg -n "state:|reconciliation:|takeover:|transfer:" examples docs --glob '!docs/roadmap/**'`
|
|
||||||
- `rg -n "single_owner|shared_root|schema version \`2\`|schema version \`3\`" docs --glob '!docs/roadmap/**'`
|
|
||||||
- `rg -n "replace_older|replace_newer|replace_conflict|replace_takeover|takeover_mode" docs README.md examples --glob '!docs/roadmap/**'`
|
|
||||||
- `rg -n "workflow: additive|workflow: replacement|schema_version.*4" docs examples`
|
|
||||||
|
|
||||||
Completion criteria:
|
|
||||||
|
|
||||||
- Current docs describe implemented catalog behavior only.
|
|
||||||
- Copyable examples load successfully.
|
|
||||||
- No current doc tells users to configure rejected legacy fields.
|
|
||||||
- Destination state documentation is schema version `4` first.
|
|
||||||
|
|
||||||
## Stage 3: Catalog Idempotent Skip
|
|
||||||
|
|
||||||
Goal: make `skip_same` a real catalog no-op optimization instead of a stale
|
|
||||||
legacy action label.
|
|
||||||
|
|
||||||
Implementation scope:
|
|
||||||
|
|
||||||
- Implement `skip_same` when every planned output is already catalog-managed
|
|
||||||
with matching:
|
|
||||||
- pipeline id;
|
|
||||||
- destination id;
|
|
||||||
- source id;
|
|
||||||
- source digest;
|
|
||||||
- source created timestamp;
|
|
||||||
- output path;
|
|
||||||
- output kind;
|
|
||||||
- output digest;
|
|
||||||
- output size;
|
|
||||||
- generated output `source_path`;
|
|
||||||
- generated output `transform`;
|
|
||||||
- output URL metadata.
|
|
||||||
- Do not read destination bytes for this optimization. Trust valid catalog
|
|
||||||
metadata; bitrot detection remains a separate future concern.
|
|
||||||
- For `workflow: additive`, allow `skip_same` when all planned outputs match and
|
|
||||||
no planned output needs to be written. Unrelated catalog outputs are ignored
|
|
||||||
for the skip decision and remain retained.
|
|
||||||
- For `workflow: replacement`, allow `skip_same` only when all planned outputs
|
|
||||||
match and there are no current-owner catalog outputs that replacement workflow
|
|
||||||
would delete.
|
|
||||||
- Never return `skip_same` for superseded legacy state, invalid state, unmanaged
|
|
||||||
content, future state, or force replacement.
|
|
||||||
- `skip_same` must not write outputs, rewrite state, delete files, or notify.
|
|
||||||
- Count `skip_same` in text and JSON summaries.
|
|
||||||
- Keep behavior deterministic and identical across local, SSH, and S3
|
|
||||||
destinations.
|
|
||||||
|
|
||||||
Current-behavior documentation updates:
|
|
||||||
|
|
||||||
- Update `docs/cli.md`, `docs/operations.md`, and `docs/internal/publish.md` to
|
|
||||||
describe catalog `skip_same`.
|
|
||||||
- Document that `skip_same` is metadata-based and does not validate destination
|
|
||||||
bytes.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- `go test ./internal/publish ./internal/app ./internal/cli`
|
|
||||||
- Repeated additive publish with identical catalog metadata returns
|
|
||||||
`skip_same`.
|
|
||||||
- Repeated replacement publish with identical catalog metadata and no omitted
|
|
||||||
current-owner outputs returns `skip_same`.
|
|
||||||
- Changed source digest, output digest, URL, transform mode, owner, output size,
|
|
||||||
or generated source path prevents skip.
|
|
||||||
- Replacement workflow does not skip when it would delete omitted current-owner
|
|
||||||
outputs.
|
|
||||||
- `skip_same` does not write outputs or state and does not notify.
|
|
||||||
- `go test ./...`
|
|
||||||
|
|
||||||
Completion criteria:
|
|
||||||
|
|
||||||
- Catalog action vocabulary has no stale action label.
|
|
||||||
- Repeated publish behavior is intentional, documented, and tested.
|
|
||||||
|
|
||||||
## Stage 4: Legacy Publish And State Code Removal
|
|
||||||
|
|
||||||
Goal: remove dead or near-dead single-owner/shared-root write and planning code
|
|
||||||
after catalog force behavior, docs, and skip semantics are settled.
|
|
||||||
|
|
||||||
Implementation scope:
|
|
||||||
|
|
||||||
- Remove legacy publish actions that are no longer planned or documented:
|
|
||||||
- `replace_older`;
|
|
||||||
- `replace_newer`;
|
|
||||||
- `replace_conflict`;
|
|
||||||
- `replace_takeover`;
|
|
||||||
- `skip_destination_newer`.
|
|
||||||
- Keep only catalog-era actions:
|
|
||||||
- `publish_new`;
|
|
||||||
- `upsert_additive`;
|
|
||||||
- `replace_catalog`;
|
|
||||||
- `skip_same`;
|
|
||||||
- `force_replace`;
|
|
||||||
- `fail_unmanaged`;
|
|
||||||
- `fail_conflict`.
|
|
||||||
- Remove legacy fields from `publish.Plan`:
|
|
||||||
- `StateMode`;
|
|
||||||
- `Reconciliation`;
|
|
||||||
- `TakeoverMode`;
|
|
||||||
- `ExistingState`;
|
|
||||||
- `ExistingSharedRoot`;
|
|
||||||
- shared-root owner/output carry fields retained only for old execution.
|
|
||||||
- Remove single-owner and shared-root execution branches from
|
|
||||||
`internal/publish/execute.go`.
|
|
||||||
- Remove unused single-owner/shared-root helper functions from `internal/publish`
|
|
||||||
once no tests or callers use them.
|
|
||||||
- Remove legacy destination config internals from `internal/config`:
|
|
||||||
- `StatePolicy`;
|
|
||||||
- `ReconciliationPolicy`;
|
|
||||||
- `TakeoverPolicy`;
|
|
||||||
- `TransferPolicy`;
|
|
||||||
- legacy constants and defaults that are no longer referenced.
|
|
||||||
- In `internal/state`, keep only what is needed for:
|
|
||||||
- parsing schema version `4` catalog state;
|
|
||||||
- identifying schema versions `1`, `2`, and `3` as superseded legacy by
|
|
||||||
schema number;
|
|
||||||
- validating and writing catalog state;
|
|
||||||
- pruning and reconciling catalog outputs.
|
|
||||||
- Delete or rewrite tests that assert legacy state parsing, validation,
|
|
||||||
comparison, shared-root ownership, or single-owner output behavior.
|
|
||||||
- Preserve test fixtures only where they are used to create superseded legacy
|
|
||||||
state for first catalog-run behavior. Prefer small local helpers over keeping
|
|
||||||
broad legacy state builders.
|
|
||||||
- Remove stale skipped tests that only represent old behavior. If a skipped test
|
|
||||||
still represents current expected behavior, unskip and update it.
|
|
||||||
|
|
||||||
Current-behavior documentation updates:
|
|
||||||
|
|
||||||
- No new user docs should be needed if Stage 2 is complete.
|
|
||||||
- Update internal docs only if removal changes internal package contracts beyond
|
|
||||||
what Stage 2 already documented.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- `go test ./internal/state`
|
|
||||||
- `go test ./internal/publish`
|
|
||||||
- `go test ./internal/app ./internal/cli`
|
|
||||||
- `go test ./internal/config`
|
|
||||||
- `go test ./...`
|
|
||||||
- Consistency searches:
|
|
||||||
- `rg -n "ActionReplaceOlder|ActionReplaceNewer|ActionReplaceConflict|ActionReplaceTakeover|ActionSkipDestinationNewer" internal`
|
|
||||||
- `rg -n "StateModeSingleOwner|StateModeSharedRoot|SharedRootState|DistributorState|ParseSharedRoot|ReconciliationPolicy|TakeoverPolicy|TransferPolicy" internal --glob '!**/*_test.go'`
|
|
||||||
- `rg -n "Retained while the executor is migrated to catalog state" internal`
|
|
||||||
|
|
||||||
Completion criteria:
|
|
||||||
|
|
||||||
- Production publish execution has one catalog code path plus explicit
|
|
||||||
`force_replace`.
|
|
||||||
- Legacy config policy types are gone from production config structs/defaults.
|
|
||||||
- State package no longer exposes full v2/v3 implementation machinery unless it
|
|
||||||
is required by tests that generate superseded legacy fixtures.
|
|
||||||
- Full test suite passes without skipped tests that mask catalog cleanup work.
|
|
||||||
|
|
||||||
## Stage 5: Roadmap Closeout
|
|
||||||
|
|
||||||
Goal: leave `docs/roadmap/` in a clean state after the follow-up work is
|
|
||||||
implemented.
|
|
||||||
|
|
||||||
Implementation scope:
|
|
||||||
|
|
||||||
- Remove or rewrite completed roadmap material:
|
|
||||||
- this `implementation.md`;
|
|
||||||
- `docs/roadmap/catalog.md`, if catalog behavior is fully documented in
|
|
||||||
current docs;
|
|
||||||
- any future roadmap entries that still describe completed catalog cleanup as
|
|
||||||
pending work.
|
|
||||||
- Keep only genuinely future work in `docs/roadmap/future.md` or another active
|
|
||||||
roadmap file.
|
|
||||||
- Ensure future work remains out of current behavior docs.
|
|
||||||
|
|
||||||
Tests and checks:
|
|
||||||
|
|
||||||
- `go test ./...`
|
|
||||||
- `git status --short`
|
|
||||||
- `rg -n "future catalog|planned catalog|single_owner|shared_root|takeover|reconciliation|transfer" docs README.md examples`
|
|
||||||
- `rg -n "workflow: additive|workflow: replacement|schema_version.*4" docs README.md examples`
|
|
||||||
|
|
||||||
Completion criteria:
|
|
||||||
|
|
||||||
- Current docs, examples, and code describe the same implemented behavior.
|
|
||||||
- Roadmap docs contain only future work.
|
|
||||||
- Full test suite passes.
|
|
||||||
|
|
||||||
## Refactors To Avoid
|
|
||||||
|
|
||||||
- Do not reintroduce `state`, `reconciliation`, `takeover`, or `transfer` YAML
|
|
||||||
fields as deprecated aliases.
|
|
||||||
- Do not migrate legacy destination state into catalog state.
|
|
||||||
- Do not add top-level `owners` or `sources` to catalog state.
|
|
||||||
- Do not persist `workflow` in `.distributor.json`.
|
|
||||||
- Do not add new prune/reconcile selectors as part of this cleanup.
|
|
||||||
- Do not move catalog or force policy into storage adapters.
|
|
||||||
- Do not redesign the run JSON envelope while cleaning action labels.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
No open questions remain. This roadmap locks the follow-up decisions:
|
|
||||||
catalog `--force` will be implemented for exceptional destructive replacement,
|
|
||||||
current docs and examples will be synchronized to catalog schema version `4`,
|
|
||||||
catalog `skip_same` will be implemented as a metadata-only no-op optimization,
|
|
||||||
and legacy single-owner/shared-root implementation debt will be removed after
|
|
||||||
the catalog action vocabulary is complete.
|
|
||||||
@@ -24,7 +24,9 @@ Reference: [Configuration](config.md#config-file-loading).
|
|||||||
|
|
||||||
Symptom: `parse config ... field not found`.
|
Symptom: `parse config ... field not found`.
|
||||||
|
|
||||||
Likely cause: the YAML contains a key that is not part of the implemented config schema.
|
Likely cause: the YAML contains a key that is not part of the implemented
|
||||||
|
config schema. Pre-workflow destination policy keys for state mode, conflict
|
||||||
|
handling, ownership adoption, or per-comparison copy decisions are rejected.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
@@ -32,9 +34,9 @@ Diagnostic:
|
|||||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: remove or rename unsupported fields using the canonical config reference.
|
Safe fix: remove unsupported fields using the canonical config reference. Destination behavior is configured with `workflow`, `publish`, `transform`, `path_mapping`, `links`, and `retention`.
|
||||||
|
|
||||||
Reference: [Configuration](config.md).
|
Reference: [Configuration](config.md#destination-fields).
|
||||||
|
|
||||||
## Backend Name Or Placement Is Invalid
|
## Backend Name Or Placement Is Invalid
|
||||||
|
|
||||||
@@ -71,9 +73,9 @@ Reference: [CLI](cli.md#validate).
|
|||||||
|
|
||||||
## Reconcile-State Selector Is Missing Or Wrong
|
## Reconcile-State Selector Is Missing Or Wrong
|
||||||
|
|
||||||
Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
|
Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "<id>" not found`, or `destination <id> not found`.
|
||||||
|
|
||||||
Likely cause: the command did not identify one configured destination root, or the selected root contains state for a different single-owner pipeline/destination.
|
Likely cause: the command did not identify one configured destination root.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
@@ -83,15 +85,15 @@ rg -n 'pipelines:|destinations:|id:' <config-path>
|
|||||||
cat <destination-path>/.distributor.json
|
cat <destination-path>/.distributor.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. For unrelated single-owner state, use the correct config selector or a separate destination root; `reconcile-state` will not take over mismatched state.
|
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file.
|
||||||
|
|
||||||
Reference: [CLI](cli.md#reconcile-state).
|
Reference: [CLI](cli.md#reconcile-state).
|
||||||
|
|
||||||
## Prune Selector Or Mode Is Missing Or Wrong
|
## Prune Selector Or Mode Is Missing Or Wrong
|
||||||
|
|
||||||
Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
|
Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "<id>" not found`, or `destination <id> not found`.
|
||||||
|
|
||||||
Likely cause: the command did not identify one configured destination root, did not choose exactly one execution mode, or the selected root contains state for a different single-owner pipeline/destination.
|
Likely cause: the command did not identify one configured destination root or did not choose exactly one execution mode.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
@@ -101,7 +103,7 @@ rg -n 'retention:|prune:|pipelines:|destinations:|id:' <config-path>
|
|||||||
cat <destination-path>/.distributor.json
|
cat <destination-path>/.distributor.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both. For unrelated single-owner state, use the correct config selector or a separate destination root; `prune` will not take over mismatched state.
|
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both.
|
||||||
|
|
||||||
Reference: [CLI](cli.md#prune).
|
Reference: [CLI](cli.md#prune).
|
||||||
|
|
||||||
@@ -187,9 +189,9 @@ Reference: [Operations](operations.md#cleanup-and-recovery).
|
|||||||
|
|
||||||
## Destination Has Unmanaged Content
|
## Destination Has Unmanaged Content
|
||||||
|
|
||||||
Symptom: `destination has content but no distributor state` or a plan reason containing `fail_unmanaged`.
|
Symptom: `destination has content but no distributor state`, `destination output path ... exists but is not managed by catalog state`, or a plan reason containing `fail_unmanaged`.
|
||||||
|
|
||||||
Likely cause: the destination bundle path contains files but no valid `.distributor.json`, so `distributor` will not claim it by default.
|
Likely cause: the destination bundle path contains files but no valid `.distributor.json`, or a planned output path collides with storage content that valid catalog state does not record. `distributor` will not claim unmanaged content by default.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
@@ -197,26 +199,43 @@ Diagnostic:
|
|||||||
find <destination-path> -maxdepth 2 -print
|
find <destination-path> -maxdepth 2 -print
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: choose an empty destination path, move unrelated files aside, or preview `run --dry-run --force` only after confirming the reported bundle path is safe to replace.
|
Safe fix: choose an empty destination path, move unrelated files aside, or preview `run --dry-run --force` only after confirming the reported destination bundle path is safe to replace.
|
||||||
|
|
||||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||||
|
|
||||||
## Destination State Conflicts With Source
|
## Destination State Is Invalid Or Unsupported
|
||||||
|
|
||||||
Symptom: `fail_conflict`, `destination source id differs`, `same id and created time but different digest`, `pipeline id ... does not match`, or `destination id ... does not match`.
|
Symptom: `fail_conflict`, `parse distributor state`, `state schema_version must be 4`, or `unsupported future destination state`.
|
||||||
|
|
||||||
Likely cause: `.distributor.json` belongs to a different pipeline, destination, source id, shared-root output owner, or same-created source with different content. Valid managed identity, source, and shared-root output-owner mismatches can publish as `replace_takeover` when destination `takeover.mode` allows them, or as `replace_conflict` when `transfer.on_conflict: replace` allows managed conflict replacement.
|
Likely cause: `.distributor.json` is invalid JSON, has invalid catalog fields, or uses an unsupported future schema.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
cat <destination-path>/.distributor.json
|
cat <destination-path>/.distributor.json
|
||||||
go run ./cmd/distributor inspect <source-root>
|
go run ./cmd/distributor run --config <config-path> --dry-run --format json
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: verify the source and destination are intended to match. Use a separate destination path for unrelated content. For normal managed replacement, configure destination `takeover.mode` to match the intended ownership boundary or configure `transfer.on_conflict: replace`, then preview with `--dry-run`. Use `--force` only for exceptional replacement of unmanaged content or other force-only cases reported as `force_replace`.
|
Safe fix: restore a valid catalog state file from backup, choose a different destination path, or use `--force` only after `run --dry-run --force` reports the intended bounded `force_replace`.
|
||||||
|
|
||||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||||
|
|
||||||
|
## Destination Uses Superseded Legacy State
|
||||||
|
|
||||||
|
Symptom: dry-run reports a normal catalog action against an older `.distributor.json`, or `reconcile-state` / `prune` reports that the destination state schema is superseded.
|
||||||
|
|
||||||
|
Likely cause: the destination contains a state file written by an older implementation. Publish planning can replace it with catalog state on successful `run`, but maintenance commands only operate on current catalog state.
|
||||||
|
|
||||||
|
Diagnostic:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cat <destination-path>/.distributor.json
|
||||||
|
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix: preview the publish plan, then run publication if the destination path is correct. The successful run writes schema version `4` catalog state.
|
||||||
|
|
||||||
|
Reference: [Destination State Contract](integrations/destination-state.md).
|
||||||
|
|
||||||
## Destination State References Missing Managed Outputs
|
## Destination State References Missing Managed Outputs
|
||||||
|
|
||||||
@@ -242,7 +261,7 @@ Reference: [Operations](operations.md#destination-state-repair).
|
|||||||
|
|
||||||
Symptom: `prune --dry-run` reports `planned=0` or JSON `planned_outputs: []`.
|
Symptom: `prune --dry-run` reports `planned=0` or JSON `planned_outputs: []`.
|
||||||
|
|
||||||
Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected shared-root owner has no eligible outputs.
|
Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected pipeline/destination owner has no eligible outputs.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
@@ -260,27 +279,11 @@ Safe fix: verify the selected destination's retention policy and the managed out
|
|||||||
|
|
||||||
Reference: [Operations](operations.md#managed-output-pruning).
|
Reference: [Operations](operations.md#managed-output-pruning).
|
||||||
|
|
||||||
## Destination Is Newer Than Source
|
|
||||||
|
|
||||||
Symptom: `skip_destination_newer`.
|
|
||||||
|
|
||||||
Likely cause: the destination state records a source manifest with a later `created` timestamp than the current source.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go run ./cmd/distributor run --config <config-path> --dry-run --format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: keep the default skip behavior unless replacement is intentional. To replace newer valid managed state, configure `transfer.on_destination_newer: replace`, preview with `--dry-run`, then publish without `--force`.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
|
||||||
|
|
||||||
## Forced Replacement Appears In A Plan
|
## Forced Replacement Appears In A Plan
|
||||||
|
|
||||||
Symptom: dry-run output includes `force_replace`.
|
Symptom: dry-run output includes `force_replace`.
|
||||||
|
|
||||||
Likely cause: the run used `--force`, and planning selected a supported destructive replacement.
|
Likely cause: the run used `--force`, and catalog planning selected a supported destructive replacement for unmanaged destination content, a planned unmanaged path collision, invalid destination state, or unsupported future destination state.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
@@ -288,15 +291,15 @@ Diagnostic:
|
|||||||
go run ./cmd/distributor run --config <config-path> --dry-run --force
|
go run ./cmd/distributor run --config <config-path> --dry-run --force
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: inspect the pipeline id, destination id, backend, and bundle path. Proceed only if deleting everything inside that destination bundle path is intended.
|
Safe fix: inspect the pipeline id, destination id, backend, and bundle path. Confirm the `destination_path` in JSON output, or the fixed-path `target=.` text output, before applying. Proceed only if deleting everything inside that destination bundle path is intended; `force_replace` then writes planned outputs and schema version `4` catalog state.
|
||||||
|
|
||||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||||
|
|
||||||
## Output Path Collision
|
## Output Path Collision
|
||||||
|
|
||||||
Symptom: `destination output path collision` or `merge output path ... exists but is not managed by destination state`.
|
Symptom: `destination output path collision` or `destination output path ... exists but is not managed by catalog state`.
|
||||||
|
|
||||||
Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`. For merge reconciliation, it can also mean a planned output path already exists in storage but is not recorded in `.distributor.json` as managed.
|
Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`, or a planned output path already exists in storage but is not catalog-managed.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
@@ -304,44 +307,10 @@ Diagnostic:
|
|||||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For merge reconciliation, move unmanaged content aside, choose another destination path, or use replacement/forced replacement only when deleting the destination bundle path is intended.
|
Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For unmanaged storage collisions, move the unmanaged file aside, choose another destination path, or use forced replacement only when deleting the destination bundle path is intended.
|
||||||
|
|
||||||
Reference: [Configuration](config.md#publish-and-transform-policy).
|
Reference: [Configuration](config.md#publish-and-transform-policy).
|
||||||
|
|
||||||
## Shared-Root Ownership Conflict
|
|
||||||
|
|
||||||
Symptom: `fail_conflict` with a reason like `destination output path ... is owned by <pipeline>/<destination>`.
|
|
||||||
|
|
||||||
Likely cause: a `state.mode: shared_root` destination planned an output path already recorded in `.distributor.json` for another pipeline/destination owner, and `takeover.mode` does not allow that managed path to move to the current owner.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
cat <destination-path>/.distributor.json
|
|
||||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: change one owner so it writes a different output path, use a separate destination root, or intentionally replace the whole destination root only after previewing with `--dry-run --force`.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
|
||||||
|
|
||||||
## Shared-Root Unmanaged Path Collision
|
|
||||||
|
|
||||||
Symptom: `fail_unmanaged` with a reason like `destination output path ... exists but is not managed by destination state`.
|
|
||||||
|
|
||||||
Likely cause: a `state.mode: shared_root` destination planned a new output path that already exists in storage but is not recorded as managed in `.distributor.json`.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
find <destination-path> -maxdepth 2 -print
|
|
||||||
cat <destination-path>/.distributor.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: move the unmanaged file aside, change the planned output path, or use forced replacement only when deleting the configured destination root is intended.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
|
||||||
|
|
||||||
## Run Failed After Writing Some Files
|
## Run Failed After Writing Some Files
|
||||||
|
|
||||||
Symptom: a destination write failed and the command exited non-zero after partial work.
|
Symptom: a destination write failed and the command exited non-zero after partial work.
|
||||||
@@ -354,9 +323,9 @@ Diagnostic:
|
|||||||
find <destination-path> -maxdepth 2 -print
|
find <destination-path> -maxdepth 2 -print
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. In same-source merge mode, previously managed retained or overwritten outputs may remain intentionally. Rerun `--dry-run` before publishing again.
|
Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. Rerun `--dry-run` before publishing again.
|
||||||
|
|
||||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
Reference: [Operations](operations.md#catalog-publish-behavior).
|
||||||
|
|
||||||
## JSON Run Result Has `ok: false`
|
## JSON Run Result Has `ok: false`
|
||||||
|
|
||||||
@@ -378,240 +347,14 @@ Reference: [CLI](cli.md#output-and-exit-behavior).
|
|||||||
|
|
||||||
Symptom: `load secrets directory ... no such file or directory`, `permission denied`, or `secret filename ... is invalid`.
|
Symptom: `load secrets directory ... no such file or directory`, `permission denied`, or `secret filename ... is invalid`.
|
||||||
|
|
||||||
Likely cause: `secrets.directory` points to a missing or unreadable directory, or it contains a regular file whose name is not a valid credential variable name.
|
Likely cause: `secrets.directory` points to a missing or unreadable directory, or contains a filename that cannot be used as a credential variable name.
|
||||||
|
|
||||||
Diagnostic:
|
Diagnostic:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
ls -ld <secrets-directory>
|
ls -la <secrets-directory>
|
||||||
find <secrets-directory> -maxdepth 1 -type f -printf '%f\n'
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix: mount or create the directory, adjust permissions for the service user, or rename/remove invalid secret files. Secret filenames must match `[A-Za-z_][A-Za-z0-9_]*`.
|
Safe fix: create the directory, fix permissions, or rename secret files to valid environment-variable-style names.
|
||||||
|
|
||||||
Reference: [Configuration](config.md#secrets).
|
Reference: [Configuration](config.md#secrets).
|
||||||
|
|
||||||
## Credential Variable Is Missing Or Empty
|
|
||||||
|
|
||||||
Symptom: `credential environment variable ... is not set`, `credential environment variable ... is empty`, or S3 authentication errors such as `AccessDenied`, `InvalidAccessKeyId`, or `SignatureDoesNotMatch`.
|
|
||||||
|
|
||||||
Likely cause: configured S3 credential variable names are not available through the process environment or `secrets.directory`, are empty, or do not authorize the requested bucket/prefix.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
env | cut -d= -f1 | rg '^(<access-key-variable>|<secret-key-variable>)$'
|
|
||||||
ls -l <secrets-directory>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: provide both configured S3 credential values, correct IAM/service permissions, or omit explicit credential fields to use the AWS SDK default credential chain.
|
|
||||||
|
|
||||||
Reference: [Configuration](config.md#s3-compatible-backend).
|
|
||||||
|
|
||||||
## Secret File Is Ignored In Favor Of Environment
|
|
||||||
|
|
||||||
Symptom: `secret ... ignored because the real environment already has that variable`.
|
|
||||||
|
|
||||||
Likely cause: the same credential name exists in the process environment and `secrets.directory` with different values.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
env | cut -d= -f1 | rg '^<variable-name>$'
|
|
||||||
ls -l <secrets-directory>/<variable-name>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: remove one source of the credential or intentionally keep the process environment value. `distributor` does not print either value.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#secrets-operation).
|
|
||||||
|
|
||||||
## SSH Auth Is Not Configured
|
|
||||||
|
|
||||||
Symptom: `no SSH auth methods configured`.
|
|
||||||
|
|
||||||
Likely cause: no SSH agent is available and `ssh_key_file` is missing or unreadable.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
test -n "$SSH_AUTH_SOCK" && ssh-add -l
|
|
||||||
ls -l <ssh-key-file>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: start an SSH agent with an appropriate key loaded, or configure a readable private key with `ssh_key_file`.
|
|
||||||
|
|
||||||
Reference: [Configuration](config.md#sshsftp-backend).
|
|
||||||
|
|
||||||
## SSH Host Key Fails
|
|
||||||
|
|
||||||
Symptom: `host key ... is unknown`, `known_hosts is required`, or `host key ... has changed`.
|
|
||||||
|
|
||||||
Likely cause: strict host key checking has no trusted key, `accept-new` cannot persist a new key, or the remote host key differs from the stored key.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
ls -l <known-hosts-path>
|
|
||||||
ssh-keygen -F <host> -f <known-hosts-path>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: pre-populate `known_hosts` for `strict`, configure a writable `known_hosts` path for `accept-new`, or verify the server identity before updating a changed key. Do not disable host key checking to bypass an unexpected changed key.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#sshsftp).
|
|
||||||
|
|
||||||
## S3 Prefix Is Invalid
|
|
||||||
|
|
||||||
Symptom: `prefix must be a clean relative slash-separated path`.
|
|
||||||
|
|
||||||
Likely cause: the S3 prefix contains traversal, dot segments, empty segments, or backslashes after leading and trailing slashes are trimmed.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: use a clean relative prefix such as `reports/archive`, or omit `prefix`.
|
|
||||||
|
|
||||||
Reference: [Configuration](config.md#s3-compatible-backend).
|
|
||||||
|
|
||||||
## S3 Location Or Connectivity Fails
|
|
||||||
|
|
||||||
Symptom: `NoSuchBucket`, `InvalidBucketName`, `not_found`, endpoint connection failures, or TLS/network errors.
|
|
||||||
|
|
||||||
Likely cause: endpoint, bucket, prefix, region, path-style mode, network routing, or credentials are wrong for the service.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
|
||||||
curl -I <endpoint>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: verify `endpoint`, `bucket`, `region`, `prefix`, and `force_path_style`. For S3-compatible services, keep `force_path_style: true` unless the service requires virtual-host addressing. Distributor does not provide an insecure TLS bypass setting.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#s3-compatible-storage).
|
|
||||||
|
|
||||||
## HTTP Server Cannot Bind
|
|
||||||
|
|
||||||
Symptom: `bind HTTP server ... address already in use`.
|
|
||||||
|
|
||||||
Likely cause: another process is listening on `server.http.bind`.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
ss -ltnp | rg '<port>'
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: stop the conflicting process or configure a different bind address.
|
|
||||||
|
|
||||||
Reference: [Configuration](config.md#serverhttp).
|
|
||||||
|
|
||||||
## HTTP Upload Token Is Missing Or Duplicated
|
|
||||||
|
|
||||||
Symptom: `upload token environment variable ... is not set`, `... is empty`, or `upload token environment variables ... resolve to the same value`.
|
|
||||||
|
|
||||||
Likely cause: a top-level upload token record references a missing or empty `token_env`, or two token records resolve to the same bearer token.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
rg -n 'token_env:' <config-path>
|
|
||||||
env | cut -d= -f1 | rg '^<token-variable>$'
|
|
||||||
ls -l <secrets-directory>/<token-variable>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: provide one distinct non-empty token value per upload token record through the process environment or `secrets.directory`. Do not put literal tokens in YAML.
|
|
||||||
|
|
||||||
Reference: [Configuration](config.md#upload_tokens).
|
|
||||||
|
|
||||||
## Upload Request Is Unauthorized
|
|
||||||
|
|
||||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `401`.
|
|
||||||
|
|
||||||
Likely cause: the request lacks `Authorization: Bearer <token>`, has an empty token, or uses a token that does not match any configured upload token record.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
curl -i -X POST http://127.0.0.1:8080/v1/pipelines/<pipeline-id>/upload \
|
|
||||||
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
|
|
||||||
-H "Content-Type: application/x-tar" \
|
|
||||||
--data-binary @bundle.tar
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: use the token value resolved by the configured `token_env`. Do not include token values in logs or tickets.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#http-upload-operation).
|
|
||||||
|
|
||||||
## Upload Request Is Forbidden
|
|
||||||
|
|
||||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `403`.
|
|
||||||
|
|
||||||
Likely cause: the bearer token is valid, but its configured `allow_pipelines` list does not include the requested upload pipeline.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
rg -n 'upload_tokens:|allow_pipelines:|id:' <config-path>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: request the intended pipeline id, or update the token allowlist to include the configured `http_upload` pipeline that this producer may submit to.
|
|
||||||
|
|
||||||
Reference: [Configuration](config.md#upload_tokens).
|
|
||||||
|
|
||||||
## Upload Request Is Rejected Before A Run ID
|
|
||||||
|
|
||||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `400`, `413`, `415`, or `503`.
|
|
||||||
|
|
||||||
Likely cause: the request path has an invalid pipeline id, included a `pipeline` or `pipeline_id` query, archive content is malformed, the body exceeds size limits, content type is unsupported, or the in-memory upload queue is full.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
tar -tf bundle.tar
|
|
||||||
tar -tzf bundle.tar.gz
|
|
||||||
rg -n 'max_upload_size|queue_size|max_concurrency' <config-path>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: send one valid tar or tar.gz source bundle archive to `/v1/pipelines/<pipeline-id>/upload` with `Content-Type: application/x-tar`, `application/gzip`, or `application/x-gzip`; remove pipeline query parameters; reduce archive size or raise the configured limit; retry after queue pressure drops.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#http-upload-operation).
|
|
||||||
|
|
||||||
## Upload Idempotency Conflict
|
|
||||||
|
|
||||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `409`.
|
|
||||||
|
|
||||||
Likely cause: the request reused an `Idempotency-Key` for the same token id and pipeline id with a different source manifest, or another request with the same key is still being staged before its manifest is known.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
curl -i -X POST http://127.0.0.1:8080/v1/pipelines/<pipeline-id>/upload \
|
|
||||||
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
|
|
||||||
-H "Content-Type: application/gzip" \
|
|
||||||
-H "Idempotency-Key: <key>" \
|
|
||||||
--data-binary @bundle.tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: if the response includes `"retryable":true`, retry the same upload later with the same key. Otherwise, inspect the producer operation and use the same key only for the same source bundle.
|
|
||||||
|
|
||||||
Reference: [HTTP Upload API Contract](integrations/http-upload.md).
|
|
||||||
|
|
||||||
## Upload Status Is Missing
|
|
||||||
|
|
||||||
Symptom: `GET /runs/<run_id>` returns `404`.
|
|
||||||
|
|
||||||
Likely cause: the run id is wrong, the process restarted, or the retained status record expired after `server.http.retention`.
|
|
||||||
|
|
||||||
Diagnostic:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
curl -i http://127.0.0.1:8080/runs/<run-id>
|
|
||||||
rg -n 'retention:' <config-path>
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix: use the exact `run_id` returned by upload admission. Increase retention if operators need a longer status window.
|
|
||||||
|
|
||||||
Reference: [Operations](operations.md#http-upload-operation).
|
|
||||||
|
|||||||
@@ -1,27 +1,25 @@
|
|||||||
pipelines:
|
pipelines:
|
||||||
- id: example-shared-root-source
|
- id: example-additive-source
|
||||||
source:
|
source:
|
||||||
backend: local
|
backend: local
|
||||||
path: examples/source-bundle
|
path: examples/source-bundle
|
||||||
destinations:
|
destinations:
|
||||||
- id: shared-local-root
|
- id: catalog-source
|
||||||
backend: local
|
backend: local
|
||||||
path: workspace/published/shared-root
|
path: workspace/published/additive-workflow
|
||||||
state:
|
workflow: additive
|
||||||
mode: shared_root
|
|
||||||
publish:
|
publish:
|
||||||
source: true
|
source: true
|
||||||
html: false
|
html: false
|
||||||
- id: example-shared-root-html
|
- id: example-additive-html
|
||||||
source:
|
source:
|
||||||
backend: local
|
backend: local
|
||||||
path: examples/source-bundle
|
path: examples/source-bundle
|
||||||
destinations:
|
destinations:
|
||||||
- id: shared-local-root
|
- id: catalog-html
|
||||||
backend: local
|
backend: local
|
||||||
path: workspace/published/shared-root
|
path: workspace/published/additive-workflow
|
||||||
state:
|
workflow: additive
|
||||||
mode: shared_root
|
|
||||||
publish:
|
publish:
|
||||||
source: false
|
source: false
|
||||||
html: true
|
html: true
|
||||||
@@ -7,6 +7,7 @@ pipelines:
|
|||||||
- id: local-source-archive
|
- id: local-source-archive
|
||||||
backend: local
|
backend: local
|
||||||
path: workspace/published/archive-and-latest/archive
|
path: workspace/published/archive-and-latest/archive
|
||||||
|
workflow: additive
|
||||||
path_mapping:
|
path_mapping:
|
||||||
mode: preserve_relative
|
mode: preserve_relative
|
||||||
publish:
|
publish:
|
||||||
@@ -15,6 +16,7 @@ pipelines:
|
|||||||
- id: local-html-latest
|
- id: local-html-latest
|
||||||
backend: local
|
backend: local
|
||||||
path: workspace/published/archive-and-latest/latest
|
path: workspace/published/archive-and-latest/latest
|
||||||
|
workflow: replacement
|
||||||
path_mapping:
|
path_mapping:
|
||||||
mode: fixed
|
mode: fixed
|
||||||
links:
|
links:
|
||||||
|
|||||||
@@ -1,18 +1,19 @@
|
|||||||
pipelines:
|
pipelines:
|
||||||
- id: example-merge-reconciliation
|
- id: example-replacement-workflow
|
||||||
source:
|
source:
|
||||||
backend: local
|
backend: local
|
||||||
path: examples/source-bundle
|
path: examples/source-bundle
|
||||||
destinations:
|
destinations:
|
||||||
- id: local-merge-html
|
- id: local-latest-html
|
||||||
backend: local
|
backend: local
|
||||||
path: workspace/published/merge-reconciliation
|
path: workspace/published/replacement-workflow
|
||||||
|
workflow: replacement
|
||||||
|
path_mapping:
|
||||||
|
mode: fixed
|
||||||
publish:
|
publish:
|
||||||
source: false
|
source: false
|
||||||
html: true
|
html: true
|
||||||
transform:
|
transform:
|
||||||
markdown_to_html:
|
markdown_to_html:
|
||||||
enabled: true
|
enabled: true
|
||||||
mode: sidecar
|
mode: index
|
||||||
reconciliation:
|
|
||||||
mode: merge
|
|
||||||
@@ -265,9 +265,6 @@ func unsupportedStateDocumentError(document state.StateDocument) error {
|
|||||||
if document.SupersededLegacy != nil {
|
if document.SupersededLegacy != nil {
|
||||||
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
|
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
|
||||||
}
|
}
|
||||||
if document.SingleOwner != nil || document.SharedRoot != nil {
|
|
||||||
return fmt.Errorf("legacy destination state is not supported by this command")
|
|
||||||
}
|
|
||||||
return fmt.Errorf("destination state document is empty")
|
return fmt.Errorf("destination state document is empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,14 +12,13 @@ func shouldNotify(action publish.Action) bool {
|
|||||||
func notifyEvent(plan publish.Plan) notify.Event {
|
func notifyEvent(plan publish.Plan) notify.Event {
|
||||||
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
||||||
for _, output := range plan.Outputs {
|
for _, output := range plan.Outputs {
|
||||||
stateOutput := output.StateOutputFile()
|
|
||||||
outputs = append(outputs, notify.Output{
|
outputs = append(outputs, notify.Output{
|
||||||
Path: stateOutput.Path,
|
Path: output.DestinationPath,
|
||||||
Kind: stateOutput.Kind,
|
Kind: output.Kind,
|
||||||
SourcePath: stateOutput.SourcePath,
|
SourcePath: output.SourcePath,
|
||||||
Transform: stateOutput.Transform,
|
Transform: output.Transform,
|
||||||
SHA256: stateOutput.SHA256,
|
SHA256: output.SHA256,
|
||||||
Size: stateOutput.Size,
|
Size: output.Size,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return notify.Event{
|
return notify.Event{
|
||||||
|
|||||||
@@ -220,15 +220,14 @@ func errorAction(pipelineID, destinationID, backend, bundlePath string, err erro
|
|||||||
func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
|
func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
|
||||||
results := make([]RunOutputRecord, 0, len(outputs))
|
results := make([]RunOutputRecord, 0, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
stateOutput := output.StateOutputFile()
|
|
||||||
results = append(results, RunOutputRecord{
|
results = append(results, RunOutputRecord{
|
||||||
Path: stateOutput.Path,
|
Path: output.DestinationPath,
|
||||||
Kind: stateOutput.Kind,
|
Kind: output.Kind,
|
||||||
SourcePath: stateOutput.SourcePath,
|
SourcePath: output.SourcePath,
|
||||||
Transform: stateOutput.Transform,
|
Transform: output.Transform,
|
||||||
URL: stateOutput.URL,
|
URL: output.URL,
|
||||||
SHA256: stateOutput.SHA256,
|
SHA256: output.SHA256,
|
||||||
Size: stateOutput.Size,
|
Size: output.Size,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return results
|
return results
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func (s *runSummary) recordPlan(action publish.Action) {
|
|||||||
s.replaceCatalog++
|
s.replaceCatalog++
|
||||||
case publish.ActionForceReplace:
|
case publish.ActionForceReplace:
|
||||||
s.forceReplace++
|
s.forceReplace++
|
||||||
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
case publish.ActionSkipSame:
|
||||||
s.skipSame++
|
s.skipSame++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -210,14 +210,14 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
|||||||
if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
|
if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
|
||||||
t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
|
t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
|
||||||
}
|
}
|
||||||
if destinationState.Source.Manifest.ID != manifest.ID {
|
if destinationState.SourceID != manifest.ID {
|
||||||
t.Fatalf("state source id = %q, want %q", destinationState.Source.Manifest.ID, manifest.ID)
|
t.Fatalf("state source id = %q, want %q", destinationState.SourceID, manifest.ID)
|
||||||
}
|
}
|
||||||
if got, want := len(destinationState.Outputs), 2; got != want {
|
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||||
t.Fatalf("state output count = %d, want %d", got, want)
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
}
|
}
|
||||||
if destinationState.Links != nil || destinationState.Outputs[0].URL != "" {
|
if destinationState.PrimaryURL != "" || destinationState.Outputs[0].URL != "" {
|
||||||
t.Fatalf("state links = %#v output URL=%q, want absent", destinationState.Links, destinationState.Outputs[0].URL)
|
t.Fatalf("state primary URL = %q output URL=%q, want absent", destinationState.PrimaryURL, destinationState.Outputs[0].URL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,8 +362,8 @@ func TestRunRecordsLinksForNestedBundlePath(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName))
|
||||||
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
if destinationState.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||||
t.Fatalf("state links = %#v, want source primary URL", destinationState.Links)
|
t.Fatalf("state primary URL = %q, want source primary URL", destinationState.PrimaryURL)
|
||||||
}
|
}
|
||||||
outputs := outputsByPath(destinationState.Outputs)
|
outputs := outputsByPath(destinationState.Outputs)
|
||||||
if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" {
|
||||||
@@ -387,8 +387,8 @@ func TestRunRecordsLinksForFixedIndexDestination(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/latest/" {
|
if destinationState.PrimaryURL != "https://reports.example.com/latest/" {
|
||||||
t.Fatalf("state links = %#v, want fixed index primary URL", destinationState.Links)
|
t.Fatalf("state primary URL = %q, want fixed index primary URL", destinationState.PrimaryURL)
|
||||||
}
|
}
|
||||||
if got, want := len(destinationState.Outputs), 1; got != want {
|
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||||
t.Fatalf("state output count = %d, want %d", got, want)
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
@@ -427,8 +427,8 @@ func TestRunFixedPathPublishesNewestBundleAtDestinationRoot(t *testing.T) {
|
|||||||
t.Fatalf("nested new report stat error = %v, want not exist", err)
|
t.Fatalf("nested new report stat error = %v, want not exist", err)
|
||||||
}
|
}
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
if destinationState.Source.Manifest.ID != "reports.new" {
|
if destinationState.SourceID != "reports.new" {
|
||||||
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
|
t.Fatalf("state source id = %q, want reports.new", destinationState.SourceID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -457,8 +457,8 @@ func TestRunFixedPathTieBreaksByBundlePath(t *testing.T) {
|
|||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
if destinationState.Source.Manifest.ID != "reports.a" {
|
if destinationState.SourceID != "reports.a" {
|
||||||
t.Fatalf("state source id = %q, want reports.a", destinationState.Source.Manifest.ID)
|
t.Fatalf("state source id = %q, want reports.a", destinationState.SourceID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -597,7 +597,6 @@ func TestRunJSONIncludesWorkflowActionAndSummary(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
|
func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
|
||||||
t.Skip("catalog workflow protects planned path collisions rather than unrelated unplanned content")
|
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{})
|
writeSourceBundle(t, sourceRoot, "bundle", testBundleOptions{})
|
||||||
@@ -612,7 +611,6 @@ func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
|
func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
|
||||||
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
|
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
parent := t.TempDir()
|
parent := t.TempDir()
|
||||||
destinationRoot := filepath.Join(parent, "latest")
|
destinationRoot := filepath.Join(parent, "latest")
|
||||||
@@ -639,6 +637,10 @@ func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
|
|||||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
||||||
t.Fatalf("unmanaged stat error = %v, want removed", err)
|
t.Fatalf("unmanaged stat error = %v, want removed", err)
|
||||||
}
|
}
|
||||||
|
catalog := readLocalCatalogState(t, destinationRoot)
|
||||||
|
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||||
|
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
|
func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
|
||||||
@@ -779,6 +781,13 @@ func TestRunNotifiesAfterReplacement(t *testing.T) {
|
|||||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
t.Fatalf("first Run() error = %v", err)
|
t.Fatalf("first Run() error = %v", err)
|
||||||
}
|
}
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||||
|
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||||
|
{Path: "summary.txt", Data: "New summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
notifier := &recordingNotifier{}
|
notifier := &recordingNotifier{}
|
||||||
|
|
||||||
err := Run(context.Background(), RunOptions{
|
err := Run(context.Background(), RunOptions{
|
||||||
@@ -1061,6 +1070,13 @@ func TestRunNotifiesForAdditiveUpsert(t *testing.T) {
|
|||||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
t.Fatalf("first Run() error = %v", err)
|
t.Fatalf("first Run() error = %v", err)
|
||||||
}
|
}
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||||
|
Created: testutil.DefaultCreated.Add(time.Hour),
|
||||||
|
Files: []testFile{
|
||||||
|
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||||
|
{Path: "summary.txt", Data: "New summary\n"},
|
||||||
|
},
|
||||||
|
})
|
||||||
notifier := &recordingNotifier{}
|
notifier := &recordingNotifier{}
|
||||||
|
|
||||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Notifier: notifier})
|
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Notifier: notifier})
|
||||||
@@ -1364,7 +1380,6 @@ func TestRunReplacesHTMLIndexOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||||
t.Skip("idempotent write is allowed for matching catalog outputs")
|
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
@@ -1372,6 +1387,55 @@ func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
|||||||
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
t.Fatalf("first Run() error = %v", err)
|
t.Fatalf("first Run() error = %v", err)
|
||||||
}
|
}
|
||||||
|
statePath := filepath.Join(destinationRoot, storage.StateFileName)
|
||||||
|
stateBefore, err := os.ReadFile(statePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read state before second run: %v", err)
|
||||||
|
}
|
||||||
|
reportBefore, err := os.ReadFile(filepath.Join(destinationRoot, "report.md"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read report before second run: %v", err)
|
||||||
|
}
|
||||||
|
notifier := &recordingNotifier{}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err = Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout, Notifier: notifier})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "action=skip_same") {
|
||||||
|
t.Fatalf("stdout = %q, want skip_same", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Final status: ok planned=1 publish_new=0 upsert_additive=0 replace_catalog=0 skip_same=1") {
|
||||||
|
t.Fatalf("stdout = %q, want skip_same summary", stdout.String())
|
||||||
|
}
|
||||||
|
stateAfter, err := os.ReadFile(statePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read state after second run: %v", err)
|
||||||
|
}
|
||||||
|
if string(stateAfter) != string(stateBefore) {
|
||||||
|
t.Fatalf("state changed during skip")
|
||||||
|
}
|
||||||
|
reportAfter, err := os.ReadFile(filepath.Join(destinationRoot, "report.md"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read report after second run: %v", err)
|
||||||
|
}
|
||||||
|
if string(reportAfter) != string(reportBefore) {
|
||||||
|
t.Fatalf("report changed during skip")
|
||||||
|
}
|
||||||
|
if got, want := len(notifier.events), 0; got != want {
|
||||||
|
t.Fatalf("notification count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunReplacementSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, config.WorkflowReplacement)
|
||||||
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
|
t.Fatalf("first Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout})
|
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout})
|
||||||
@@ -1384,7 +1448,6 @@ func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFailsOnUnmanagedDestination(t *testing.T) {
|
func TestRunFailsOnUnmanagedDestination(t *testing.T) {
|
||||||
t.Skip("catalog workflow protects planned path collisions rather than unrelated unplanned content")
|
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
@@ -1399,7 +1462,6 @@ func TestRunFailsOnUnmanagedDestination(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
|
func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
|
||||||
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
|
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
@@ -1419,10 +1481,50 @@ func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
|
|||||||
if !strings.Contains(stdout.String(), "action=force_replace") {
|
if !strings.Contains(stdout.String(), "action=force_replace") {
|
||||||
t.Fatalf("stdout = %q, want force_replace", stdout.String())
|
t.Fatalf("stdout = %q, want force_replace", stdout.String())
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Final status: ok planned=1 publish_new=0 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=1 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=false") {
|
||||||
|
t.Fatalf("stdout = %q, want force_replace counter only", stdout.String())
|
||||||
|
}
|
||||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
|
||||||
t.Fatalf("unmanaged file stat error = %v, want not exist", err)
|
t.Fatalf("unmanaged file stat error = %v, want not exist", err)
|
||||||
}
|
}
|
||||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
|
catalog := readLocalCatalogState(t, destinationRoot)
|
||||||
|
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||||
|
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunForceReplacementJSONOutput(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write unmanaged file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
Force: true,
|
||||||
|
Stdout: &stdout,
|
||||||
|
OutputFormat: OutputFormatJSON,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
result := decodeAppResult(t, stdout.String())
|
||||||
|
actions, ok := result["actions"].([]any)
|
||||||
|
if !ok || len(actions) != 1 {
|
||||||
|
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||||
|
}
|
||||||
|
action, ok := actions[0].(map[string]any)
|
||||||
|
if !ok || action["action"] != "force_replace" || action["workflow"] != "additive" {
|
||||||
|
t.Fatalf("action = %#v, want force_replace additive", actions[0])
|
||||||
|
}
|
||||||
|
summary, ok := result["summary"].(map[string]any)
|
||||||
|
if !ok || summary["force_replace"] != float64(1) || summary["publish_new"] != float64(0) {
|
||||||
|
t.Fatalf("summary = %#v, want force_replace only", result["summary"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFansOutToLocalDestinations(t *testing.T) {
|
func TestRunFansOutToLocalDestinations(t *testing.T) {
|
||||||
@@ -1521,13 +1623,12 @@ func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
|
|||||||
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &repeatOutput}, provider); err != nil {
|
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &repeatOutput}, provider); err != nil {
|
||||||
t.Fatalf("repeat error = %v", err)
|
t.Fatalf("repeat error = %v", err)
|
||||||
}
|
}
|
||||||
if got := strings.Count(repeatOutput.String(), "action=upsert_additive"); got != 4 {
|
if got := strings.Count(repeatOutput.String(), "action=skip_same"); got != 4 {
|
||||||
t.Fatalf("repeat output = %q, upsert_additive count = %d, want 4", repeatOutput.String(), got)
|
t.Fatalf("repeat output = %q, skip_same count = %d, want 4", repeatOutput.String(), got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
|
func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
|
||||||
t.Skip("force reporting and execution behavior is covered by the catalog reporting work")
|
|
||||||
localSourceRoot := t.TempDir()
|
localSourceRoot := t.TempDir()
|
||||||
writeSourceBundle(t, localSourceRoot, "bundle", testBundleOptions{})
|
writeSourceBundle(t, localSourceRoot, "bundle", testBundleOptions{})
|
||||||
s3Destination := fake.New()
|
s3Destination := fake.New()
|
||||||
@@ -1569,6 +1670,12 @@ func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
|
|||||||
testutil.AssertFakeFile(t, sshDestination, "bundle/report.md", "# Report\nSunny.\n")
|
testutil.AssertFakeFile(t, sshDestination, "bundle/report.md", "# Report\nSunny.\n")
|
||||||
testutil.AssertFakeMissing(t, sshDestination, "bundle/old.txt")
|
testutil.AssertFakeMissing(t, sshDestination, "bundle/old.txt")
|
||||||
testutil.AssertFakeFile(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
testutil.AssertFakeFile(t, sshDestination, "bundle-sibling/keep.txt", "keep")
|
||||||
|
for name, backend := range map[string]*fake.Backend{"s3": s3Destination, "ssh": sshDestination} {
|
||||||
|
catalog := readFakeCatalogStateAt(t, backend, "bundle")
|
||||||
|
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||||
|
t.Fatalf("%s catalog identity = schema %d mode %s", name, catalog.SchemaVersion, catalog.State.Mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunDryRunDoesNotWrite(t *testing.T) {
|
func TestRunDryRunDoesNotWrite(t *testing.T) {
|
||||||
@@ -1700,11 +1807,6 @@ func writeConfigFile(t *testing.T, body string) string {
|
|||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeDestinationState(t *testing.T, root, relative string, manifest bundle.Manifest) {
|
|
||||||
t.Helper()
|
|
||||||
testutil.WriteDestinationState(t, root, relative, manifest, testutil.DestinationStateOptions{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||||
@@ -1722,8 +1824,10 @@ func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
|
|||||||
type testDestinationState struct {
|
type testDestinationState struct {
|
||||||
PipelineID string
|
PipelineID string
|
||||||
DestinationID string
|
DestinationID string
|
||||||
Source state.SourceState
|
SourceID string
|
||||||
Links *state.LinkState
|
SourceDigest string
|
||||||
|
SourceCreated time.Time
|
||||||
|
PrimaryURL string
|
||||||
Outputs []testStateOutput
|
Outputs []testStateOutput
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1752,11 +1856,11 @@ func readStateFile(t *testing.T, path string) testDestinationState {
|
|||||||
if index == 0 {
|
if index == 0 {
|
||||||
view.PipelineID = output.PipelineID
|
view.PipelineID = output.PipelineID
|
||||||
view.DestinationID = output.DestinationID
|
view.DestinationID = output.DestinationID
|
||||||
view.Source.Manifest.ID = output.Source.ID
|
view.SourceID = output.Source.ID
|
||||||
view.Source.Manifest.Digest = output.Source.Digest
|
view.SourceDigest = output.Source.Digest
|
||||||
view.Source.Manifest.Created = output.Source.Created
|
view.SourceCreated = output.Source.Created
|
||||||
if output.URL != "" {
|
if output.URL != "" {
|
||||||
view.Links = &state.LinkState{PrimaryURL: output.URL}
|
view.PrimaryURL = output.URL
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
view.Outputs = append(view.Outputs, testStateOutput{
|
view.Outputs = append(view.Outputs, testStateOutput{
|
||||||
@@ -1772,6 +1876,36 @@ func readStateFile(t *testing.T, path string) testDestinationState {
|
|||||||
return view
|
return view
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func readLocalCatalogState(t *testing.T, destinationRoot string) state.CatalogState {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(filepath.Join(destinationRoot, storage.StateFileName))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read catalog state: %v", err)
|
||||||
|
}
|
||||||
|
catalog, err := state.ParseCatalog(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse catalog state: %v", err)
|
||||||
|
}
|
||||||
|
return catalog
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFakeCatalogStateAt(t *testing.T, backend *fake.Backend, bundlePath string) state.CatalogState {
|
||||||
|
t.Helper()
|
||||||
|
statePath, err := storage.StatePath(bundlePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("state path: %v", err)
|
||||||
|
}
|
||||||
|
data, err := backend.ReadFile(context.Background(), statePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read catalog state: %v", err)
|
||||||
|
}
|
||||||
|
catalog, err := state.ParseCatalog(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse catalog state: %v", err)
|
||||||
|
}
|
||||||
|
return catalog
|
||||||
|
}
|
||||||
|
|
||||||
func outputsByPath(outputs []testStateOutput) map[string]testStateOutput {
|
func outputsByPath(outputs []testStateOutput) map[string]testStateOutput {
|
||||||
byPath := make(map[string]testStateOutput, len(outputs))
|
byPath := make(map[string]testStateOutput, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
|
|||||||
@@ -693,6 +693,42 @@ func TestExecuteRunJSONDryRun(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunJSONReportsSkipSame(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||||
|
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||||
|
|
||||||
|
var firstStdout, firstStderr bytes.Buffer
|
||||||
|
if code := Execute(context.Background(), []string{"run", "--config", configPath}, &firstStdout, &firstStderr); code != exitOK {
|
||||||
|
t.Fatalf("first exit code = %d, want %d; stderr = %q", code, exitOK, firstStderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", configPath, "--format", "json"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
envelope := decodeEnvelope(t, &stdout)
|
||||||
|
result := envelopeResult(t, envelope)
|
||||||
|
actions, ok := result["actions"].([]any)
|
||||||
|
if !ok || len(actions) != 1 {
|
||||||
|
t.Fatalf("actions = %#v, want one action", result["actions"])
|
||||||
|
}
|
||||||
|
action, ok := actions[0].(map[string]any)
|
||||||
|
if !ok || action["action"] != "skip_same" {
|
||||||
|
t.Fatalf("action = %#v, want skip_same", actions[0])
|
||||||
|
}
|
||||||
|
summary, ok := result["summary"].(map[string]any)
|
||||||
|
if !ok || summary["skip_same"] != float64(1) || summary["publish_new"] != float64(0) {
|
||||||
|
t.Fatalf("summary = %#v, want skip_same counter", result["summary"])
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRunJSONDryRunReportsFixedPathMapping(t *testing.T) {
|
func TestExecuteRunJSONDryRunReportsFixedPathMapping(t *testing.T) {
|
||||||
sourceRoot := t.TempDir()
|
sourceRoot := t.TempDir()
|
||||||
destinationRoot := t.TempDir()
|
destinationRoot := t.TempDir()
|
||||||
@@ -938,8 +974,8 @@ func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
|
|||||||
if code != exitOK {
|
if code != exitOK {
|
||||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout.String(), "action=publish_new workflow=additive") {
|
if !strings.Contains(stdout.String(), "action=force_replace workflow=additive") || !strings.Contains(stdout.String(), "force_replace=1") {
|
||||||
t.Fatalf("stdout = %q, want additive publish", stdout.String())
|
t.Fatalf("stdout = %q, want forced replacement", stdout.String())
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); err != nil {
|
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); err != nil {
|
||||||
t.Fatalf("unmanaged file stat error = %v", err)
|
t.Fatalf("unmanaged file stat error = %v", err)
|
||||||
|
|||||||
@@ -56,11 +56,7 @@ type Destination struct {
|
|||||||
PathMap PathMapping `yaml:"path_mapping"`
|
PathMap PathMapping `yaml:"path_mapping"`
|
||||||
Links *Links `yaml:"links"`
|
Links *Links `yaml:"links"`
|
||||||
Workflow string `yaml:"workflow"`
|
Workflow string `yaml:"workflow"`
|
||||||
State StatePolicy `yaml:"-"`
|
|
||||||
Reconciliation ReconciliationPolicy `yaml:"-"`
|
|
||||||
Takeover TakeoverPolicy `yaml:"-"`
|
|
||||||
Retention RetentionPolicy `yaml:"retention"`
|
Retention RetentionPolicy `yaml:"retention"`
|
||||||
Transfer TransferPolicy `yaml:"-"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Backend struct {
|
type Backend struct {
|
||||||
@@ -124,18 +120,6 @@ type Links struct {
|
|||||||
Primary string `yaml:"primary"`
|
Primary string `yaml:"primary"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ReconciliationPolicy struct {
|
|
||||||
Mode string `yaml:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TakeoverPolicy struct {
|
|
||||||
Mode string `yaml:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type StatePolicy struct {
|
|
||||||
Mode string `yaml:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type RetentionPolicy struct {
|
type RetentionPolicy struct {
|
||||||
Prune PrunePolicy `yaml:"prune"`
|
Prune PrunePolicy `yaml:"prune"`
|
||||||
}
|
}
|
||||||
@@ -145,10 +129,3 @@ type PrunePolicy struct {
|
|||||||
OlderThan *Duration `yaml:"older_than"`
|
OlderThan *Duration `yaml:"older_than"`
|
||||||
KeepLatest *int `yaml:"keep_latest"`
|
KeepLatest *int `yaml:"keep_latest"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TransferPolicy struct {
|
|
||||||
OnDestinationSame string `yaml:"on_destination_same"`
|
|
||||||
OnDestinationOlder string `yaml:"on_destination_older"`
|
|
||||||
OnDestinationNewer string `yaml:"on_destination_newer"`
|
|
||||||
OnConflict string `yaml:"on_conflict"`
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -20,12 +20,6 @@ const (
|
|||||||
ValidationActionFail = "fail"
|
ValidationActionFail = "fail"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
TransferActionSkip = "skip"
|
|
||||||
TransferActionReplace = "replace"
|
|
||||||
TransferActionFail = "fail"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
TransformModeSidecar = transform.MarkdownModeSidecar
|
TransformModeSidecar = transform.MarkdownModeSidecar
|
||||||
TransformModeIndex = transform.MarkdownModeIndex
|
TransformModeIndex = transform.MarkdownModeIndex
|
||||||
@@ -47,23 +41,6 @@ const (
|
|||||||
WorkflowReplacement = "replacement"
|
WorkflowReplacement = "replacement"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
ReconciliationModeReplace = "replace"
|
|
||||||
ReconciliationModeMerge = "merge"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
TakeoverModeSamePipeline = "same_pipeline"
|
|
||||||
TakeoverModeSameSource = "same_source"
|
|
||||||
TakeoverModeAnyManaged = "any_managed"
|
|
||||||
TakeoverModeNever = "never"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
StateModeSingleOwner = "single_owner"
|
|
||||||
StateModeSharedRoot = "shared_root"
|
|
||||||
)
|
|
||||||
|
|
||||||
const DefaultS3Region = "us-east-1"
|
const DefaultS3Region = "us-east-1"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -104,27 +81,6 @@ func ApplyDefaults(cfg *Config) {
|
|||||||
if destination.Workflow == "" {
|
if destination.Workflow == "" {
|
||||||
destination.Workflow = WorkflowAdditive
|
destination.Workflow = WorkflowAdditive
|
||||||
}
|
}
|
||||||
if destination.State.Mode == "" {
|
|
||||||
destination.State.Mode = StateModeSingleOwner
|
|
||||||
}
|
|
||||||
if destination.Reconciliation.Mode == "" {
|
|
||||||
destination.Reconciliation.Mode = ReconciliationModeReplace
|
|
||||||
}
|
|
||||||
if destination.Takeover.Mode == "" {
|
|
||||||
destination.Takeover.Mode = TakeoverModeSamePipeline
|
|
||||||
}
|
|
||||||
if destination.Transfer.OnDestinationSame == "" {
|
|
||||||
destination.Transfer.OnDestinationSame = TransferActionSkip
|
|
||||||
}
|
|
||||||
if destination.Transfer.OnDestinationOlder == "" {
|
|
||||||
destination.Transfer.OnDestinationOlder = TransferActionReplace
|
|
||||||
}
|
|
||||||
if destination.Transfer.OnDestinationNewer == "" {
|
|
||||||
destination.Transfer.OnDestinationNewer = TransferActionSkip
|
|
||||||
}
|
|
||||||
if destination.Transfer.OnConflict == "" {
|
|
||||||
destination.Transfer.OnConflict = TransferActionFail
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -958,17 +958,14 @@ pipelines:
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExampleConfigsLoad(t *testing.T) {
|
func TestExampleConfigsLoad(t *testing.T) {
|
||||||
for _, path := range []string{
|
paths, err := filepath.Glob("../../examples/*.yml")
|
||||||
"../../examples/local-to-local.yml",
|
if err != nil {
|
||||||
"../../examples/local-publish.yml",
|
t.Fatalf("glob examples: %v", err)
|
||||||
"../../examples/local-html.yml",
|
}
|
||||||
"../../examples/local-index.yml",
|
if len(paths) == 0 {
|
||||||
"../../examples/fan-out.yml",
|
t.Fatal("no example configs found")
|
||||||
"../../examples/archive-and-latest.yml",
|
}
|
||||||
"../../examples/http-upload-local.yml",
|
for _, path := range paths {
|
||||||
"../../examples/ssh-destination.yml",
|
|
||||||
"../../examples/s3-destination.yml",
|
|
||||||
} {
|
|
||||||
t.Run(path, func(t *testing.T) {
|
t.Run(path, func(t *testing.T) {
|
||||||
if _, err := LoadFile(path); err != nil {
|
if _, err := LoadFile(path); err != nil {
|
||||||
t.Fatalf("LoadFile(%q) error = %v", path, err)
|
t.Fatalf("LoadFile(%q) error = %v", path, err)
|
||||||
|
|||||||
@@ -5,41 +5,23 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Execute(ctx context.Context, req Request, plan Plan) error {
|
func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
|
||||||
switch plan.Action {
|
switch plan.Action {
|
||||||
case ActionSkipSame, ActionSkipDestinationNewer:
|
case ActionSkipSame:
|
||||||
return nil
|
return nil
|
||||||
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog:
|
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog, ActionForceReplace:
|
||||||
return executeCatalog(ctx, req, plan)
|
return executeCatalog(ctx, req, plan)
|
||||||
case ActionReplaceOlder, ActionReplaceConflict, ActionReplaceNewer, ActionReplaceTakeover, ActionForceReplace:
|
|
||||||
if usesSharedRootState(req, plan) {
|
|
||||||
return executeSharedRoot(ctx, req, plan)
|
|
||||||
}
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if plan.Action == ActionReplaceOlder || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceNewer || plan.Action == ActionReplaceTakeover {
|
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
|
||||||
if plan.ExistingState == nil {
|
|
||||||
return fmt.Errorf("replace requires existing destination state")
|
|
||||||
}
|
|
||||||
if plan.Reconciliation.Mode == config.ReconciliationModeReplace || plan.Action == ActionReplaceConflict || plan.Action == ActionReplaceTakeover {
|
|
||||||
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if plan.Action == ActionForceReplace {
|
if plan.Action == ActionForceReplace {
|
||||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -47,102 +29,7 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
|||||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
} else if plan.Action == ActionReplaceCatalog {
|
||||||
if usesMergeRetention(plan) {
|
|
||||||
if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
|
||||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
|
||||||
cleanup := func() {
|
|
||||||
outputs := writtenOutputs
|
|
||||||
if usesMergeRetention(plan) {
|
|
||||||
outputs = newOutputs
|
|
||||||
}
|
|
||||||
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
|
||||||
}
|
|
||||||
for _, output := range plan.Outputs {
|
|
||||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data := output.Data
|
|
||||||
if output.Kind == state.OutputKindSource {
|
|
||||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
managed := outputManagedByExistingState(output, plan.ExistingState)
|
|
||||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
writtenOutputs = append(writtenOutputs, output)
|
|
||||||
if !managed {
|
|
||||||
newOutputs = append(newOutputs, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
createdAt := now
|
|
||||||
if plan.ExistingState != nil {
|
|
||||||
createdAt = plan.ExistingState.CreatedAt
|
|
||||||
}
|
|
||||||
stateOutputs, err := stateOutputsForPlan(plan, now)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
destinationState := state.DistributorState{
|
|
||||||
SchemaVersion: state.SchemaVersion,
|
|
||||||
DistributorVersion: req.DistributorVersion,
|
|
||||||
PipelineID: req.PipelineID,
|
|
||||||
DestinationID: req.DestinationID,
|
|
||||||
PublishedAt: now,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: now,
|
|
||||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
|
||||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
|
||||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
|
||||||
Outputs: stateOutputs,
|
|
||||||
}
|
|
||||||
if plan.PrimaryURL != "" {
|
|
||||||
destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
|
||||||
}
|
|
||||||
if err := state.Validate(destinationState); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data = append(data, '\n')
|
|
||||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
|
|
||||||
if plan.Action == ActionReplaceCatalog {
|
|
||||||
if plan.ClearDestinationRoot {
|
if plan.ClearDestinationRoot {
|
||||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -286,264 +173,3 @@ func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
|
|||||||
}
|
}
|
||||||
return paths
|
return paths
|
||||||
}
|
}
|
||||||
|
|
||||||
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
|
||||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
|
||||||
if plan.Action == ActionForceReplace {
|
|
||||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if plan.Action == ActionReplaceTakeover || plan.Action == ActionReplaceConflict || (isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeReplace) {
|
|
||||||
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
|
||||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
|
||||||
cleanup := func() {
|
|
||||||
outputs := writtenOutputs
|
|
||||||
if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
|
||||||
outputs = newOutputs
|
|
||||||
}
|
|
||||||
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
|
||||||
}
|
|
||||||
for _, output := range plan.Outputs {
|
|
||||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data := output.Data
|
|
||||||
if output.Kind == state.OutputKindSource {
|
|
||||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
managed := outputManagedBySharedRootPlan(output, plan)
|
|
||||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
writtenOutputs = append(writtenOutputs, output)
|
|
||||||
if !managed {
|
|
||||||
newOutputs = append(newOutputs, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
sharedRootState, err := sharedRootStateForPlan(req, plan, now)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := state.ValidateSharedRoot(sharedRootState); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data, err := json.MarshalIndent(sharedRootState, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
data = append(data, '\n')
|
|
||||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
|
||||||
if err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
|
|
||||||
cleanup()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error {
|
|
||||||
for _, output := range plan.Outputs {
|
|
||||||
if outputManagedByExistingState(output, plan.ExistingState) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
|
||||||
return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath))
|
|
||||||
} else if !storage.IsNotFound(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func usesSharedRootState(req Request, plan Plan) bool {
|
|
||||||
return plan.StateMode == config.StateModeSharedRoot
|
|
||||||
}
|
|
||||||
|
|
||||||
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
|
|
||||||
if existing == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
_, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath)
|
|
||||||
return ok
|
|
||||||
}
|
|
||||||
|
|
||||||
func outputManagedBySharedRootPlan(output Output, plan Plan) bool {
|
|
||||||
for _, existing := range currentOwnerSharedRootOutputs(plan) {
|
|
||||||
if existing.Path == output.DestinationPath {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, existing := range plan.TakenOverOwnerOutputs {
|
|
||||||
if existing.Path == output.DestinationPath {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) {
|
|
||||||
existingOutputs := []state.OutputFile(nil)
|
|
||||||
if plan.ExistingState != nil {
|
|
||||||
existingOutputs = plan.ExistingState.Outputs
|
|
||||||
}
|
|
||||||
planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now)
|
|
||||||
if !usesMergeRetention(plan) || plan.ExistingState == nil {
|
|
||||||
return planned, nil
|
|
||||||
}
|
|
||||||
return state.MergeOutputFiles(plan.ExistingState.Outputs, planned)
|
|
||||||
}
|
|
||||||
|
|
||||||
func usesMergeRetention(plan Plan) bool {
|
|
||||||
return plan.Reconciliation.Mode == config.ReconciliationModeMerge && isReconciliationReplacementAction(plan.Action)
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) {
|
|
||||||
now = now.UTC()
|
|
||||||
scope := plan.OwnerScope
|
|
||||||
if scope.PipelineID == "" && scope.DestinationID == "" {
|
|
||||||
scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
|
||||||
}
|
|
||||||
base := sharedRootBaseState(req, plan, now)
|
|
||||||
base = removeTakenOverSharedRootOutputs(base, plan.TakenOverOwnerOutputs)
|
|
||||||
owner := state.OwnerRecord{
|
|
||||||
Scope: scope,
|
|
||||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
|
||||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
|
||||||
}
|
|
||||||
if plan.PrimaryURL != "" {
|
|
||||||
owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
|
||||||
}
|
|
||||||
planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now)
|
|
||||||
if isReconciliationReplacementAction(plan.Action) && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
|
||||||
return state.MergeOwnerOutputs(base, scope, owner, planned)
|
|
||||||
}
|
|
||||||
return state.ReplaceOwnerOutputs(base, scope, owner, planned)
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState {
|
|
||||||
if plan.Action == ActionForceReplace {
|
|
||||||
return newSharedRootState(req, now)
|
|
||||||
}
|
|
||||||
if plan.ExistingSharedRoot != nil {
|
|
||||||
base := *plan.ExistingSharedRoot
|
|
||||||
base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...)
|
|
||||||
base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...)
|
|
||||||
base.DistributorVersion = req.DistributorVersion
|
|
||||||
base.UpdatedAt = now
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
if plan.ExistingState != nil {
|
|
||||||
base := newSharedRootState(req, now)
|
|
||||||
base.CreatedAt = plan.ExistingState.CreatedAt
|
|
||||||
base.UpdatedAt = now
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
return newSharedRootState(req, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
func removeTakenOverSharedRootOutputs(sharedRoot state.SharedRootState, takenOver []state.SharedRootOutputFile) state.SharedRootState {
|
|
||||||
if len(takenOver) == 0 {
|
|
||||||
return sharedRoot
|
|
||||||
}
|
|
||||||
paths := sharedRootOutputPathSet(takenOver)
|
|
||||||
next := sharedRoot
|
|
||||||
next.Outputs = make([]state.SharedRootOutputFile, 0, len(sharedRoot.Outputs))
|
|
||||||
for _, output := range sharedRoot.Outputs {
|
|
||||||
if _, remove := paths[output.Path]; remove {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
|
|
||||||
func newSharedRootState(req Request, now time.Time) state.SharedRootState {
|
|
||||||
return state.SharedRootState{
|
|
||||||
SchemaVersion: state.SharedRootSchemaVersion,
|
|
||||||
DistributorVersion: req.DistributorVersion,
|
|
||||||
CreatedAt: now,
|
|
||||||
UpdatedAt: now,
|
|
||||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
|
||||||
Owners: []state.OwnerRecord{},
|
|
||||||
Outputs: []state.SharedRootOutputFile{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile {
|
|
||||||
if plan.ExistingSharedRoot != nil {
|
|
||||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs))
|
|
||||||
for _, output := range plan.ExistingSharedRoot.Outputs {
|
|
||||||
if output.Owner == plan.OwnerScope {
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return outputs
|
|
||||||
}
|
|
||||||
if plan.ExistingState != nil {
|
|
||||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs))
|
|
||||||
for _, output := range plan.ExistingState.Outputs {
|
|
||||||
outputs = append(outputs, state.SharedRootOutputFile{
|
|
||||||
Path: output.Path,
|
|
||||||
Kind: output.Kind,
|
|
||||||
SourcePath: output.SourcePath,
|
|
||||||
Transform: output.Transform,
|
|
||||||
URL: output.URL,
|
|
||||||
SHA256: output.SHA256,
|
|
||||||
Size: output.Size,
|
|
||||||
Owner: plan.OwnerScope,
|
|
||||||
SourceID: plan.ExistingState.Source.Manifest.ID,
|
|
||||||
SourceDigest: plan.ExistingState.Source.Manifest.Digest,
|
|
||||||
SourceCreated: plan.ExistingState.Source.Manifest.Created,
|
|
||||||
CreatedAt: output.CreatedAt,
|
|
||||||
UpdatedAt: output.UpdatedAt,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return outputs
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string {
|
|
||||||
paths := make([]string, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootStateWriteOverwrites(plan Plan) bool {
|
|
||||||
return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ func TestExecuteReplacementDeletesCurrentOwnerAndPreservesOtherOwners(t *testing
|
|||||||
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||||
req.DestinationBundlePath = "bundle"
|
req.DestinationBundlePath = "bundle"
|
||||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
legacyState := legacyStateDocument(2)
|
||||||
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
|
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||||
@@ -110,7 +110,7 @@ func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
|
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
legacyState := legacyStateDocument(2)
|
||||||
writeJSONState(t, destinationBackend, "", legacyState)
|
writeJSONState(t, destinationBackend, "", legacyState)
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
|
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
|
||||||
@@ -131,6 +131,65 @@ func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteForceReplaceClearsDestinationBundlePathOnly(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
req.DestinationBundlePath = "bundle"
|
||||||
|
req.Force = true
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "bundle/report.md", "old report")
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "bundle-child/keep.txt", "keep")
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != ActionForceReplace {
|
||||||
|
t.Fatalf("plan action = %s, want %s", plan.Action, ActionForceReplace)
|
||||||
|
}
|
||||||
|
if err := Execute(context.Background(), req, plan); err != nil {
|
||||||
|
t.Fatalf("Execute() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
|
||||||
|
testutil.AssertFakeFile(t, destinationBackend, "bundle/summary.txt", "Summary\n")
|
||||||
|
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unplanned.txt")
|
||||||
|
testutil.AssertFakeFile(t, destinationBackend, "bundle-child/keep.txt", "keep")
|
||||||
|
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "keep")
|
||||||
|
catalog := readCatalogState(t, destinationBackend, "bundle")
|
||||||
|
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
|
||||||
|
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
|
||||||
|
}
|
||||||
|
if len(catalog.Outputs) != 2 {
|
||||||
|
t.Fatalf("catalog outputs = %#v, want planned outputs only", catalog.Outputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteForceReplaceClearsFixedDestinationRoot(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
req.DestinationBundlePath = ""
|
||||||
|
req.PathMapping = config.PathMappingFixed
|
||||||
|
req.Force = true
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "remove")
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != ActionForceReplace || storage.DisplayPath(plan.DestinationBundlePath) != "." {
|
||||||
|
t.Fatalf("plan action=%s destination=%s, want force_replace at root", plan.Action, storage.DisplayPath(plan.DestinationBundlePath))
|
||||||
|
}
|
||||||
|
if err := Execute(context.Background(), req, plan); err != nil {
|
||||||
|
t.Fatalf("Execute() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
|
||||||
|
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
|
||||||
|
testutil.AssertFakeMissing(t, destinationBackend, "unplanned.txt")
|
||||||
|
readCatalogState(t, destinationBackend, "")
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteFailedWriteDoesNotWriteCatalogState(t *testing.T) {
|
func TestExecuteFailedWriteDoesNotWriteCatalogState(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
plan, err := Build(context.Background(), req)
|
plan, err := Build(context.Background(), req)
|
||||||
|
|||||||
@@ -98,50 +98,10 @@ func rejectOutputCollisions(outputs []Output) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o Output) StateOutputFile() state.OutputFile {
|
|
||||||
return state.OutputFile{
|
|
||||||
Path: o.DestinationPath,
|
|
||||||
Kind: o.Kind,
|
|
||||||
SourcePath: o.SourcePath,
|
|
||||||
Transform: o.Transform,
|
|
||||||
URL: o.URL,
|
|
||||||
SHA256: o.SHA256,
|
|
||||||
Size: o.Size,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o Output) StateOutputProjection() state.OutputProjection {
|
|
||||||
return state.OutputProjection{
|
|
||||||
Path: o.DestinationPath,
|
|
||||||
Kind: o.Kind,
|
|
||||||
SourcePath: o.SourcePath,
|
|
||||||
Transform: o.Transform,
|
|
||||||
URL: o.URL,
|
|
||||||
SHA256: o.SHA256,
|
|
||||||
Size: o.Size,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o Output) ManagedPath() string {
|
func (o Output) ManagedPath() string {
|
||||||
return o.DestinationPath
|
return o.DestinationPath
|
||||||
}
|
}
|
||||||
|
|
||||||
func StateOutputFiles(outputs []Output) []state.OutputFile {
|
|
||||||
files := make([]state.OutputFile, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
files = append(files, output.StateOutputFile())
|
|
||||||
}
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
func StateOutputProjections(outputs []Output) []state.OutputProjection {
|
|
||||||
projections := make([]state.OutputProjection, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
projections = append(projections, output.StateOutputProjection())
|
|
||||||
}
|
|
||||||
return projections
|
|
||||||
}
|
|
||||||
|
|
||||||
func ManagedOutputPaths(outputs []Output) []string {
|
func ManagedOutputPaths(outputs []Output) []string {
|
||||||
paths := make([]string, 0, len(outputs))
|
paths := make([]string, 0, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
|
|||||||
@@ -12,46 +12,12 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestOutputStateProjection(t *testing.T) {
|
func TestManagedOutputPaths(t *testing.T) {
|
||||||
sourceOutput := Output{
|
|
||||||
SourcePath: "report.md",
|
|
||||||
DestinationPath: "report.md",
|
|
||||||
Kind: state.OutputKindSource,
|
|
||||||
URL: "https://reports.example.com/report.md",
|
|
||||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
||||||
Size: 123,
|
|
||||||
}
|
|
||||||
sourceState := sourceOutput.StateOutputFile()
|
|
||||||
if sourceState.Path != "report.md" || sourceState.Kind != state.OutputKindSource || sourceState.SourcePath != "report.md" || sourceState.URL != sourceOutput.URL || sourceState.SHA256 != sourceOutput.SHA256 || sourceState.Size != sourceOutput.Size {
|
|
||||||
t.Fatalf("source state output = %#v", sourceState)
|
|
||||||
}
|
|
||||||
|
|
||||||
generatedOutput := Output{
|
|
||||||
SourcePath: "report.md",
|
|
||||||
DestinationPath: "report.html",
|
|
||||||
Kind: state.OutputKindGenerated,
|
|
||||||
Transform: transform.MarkdownToHTML,
|
|
||||||
URL: "https://reports.example.com/report.html",
|
|
||||||
SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
||||||
Size: 456,
|
|
||||||
}
|
|
||||||
generatedState := generatedOutput.StateOutputFile()
|
|
||||||
if generatedState.Path != "report.html" || generatedState.Kind != state.OutputKindGenerated || generatedState.SourcePath != "report.md" || generatedState.Transform != transform.MarkdownToHTML || generatedState.URL != generatedOutput.URL || generatedState.SHA256 != generatedOutput.SHA256 || generatedState.Size != generatedOutput.Size {
|
|
||||||
t.Fatalf("generated state output = %#v", generatedState)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOutputSliceProjections(t *testing.T) {
|
|
||||||
outputs := []Output{
|
outputs := []Output{
|
||||||
{SourcePath: "report.md", DestinationPath: "report.md", Kind: state.OutputKindSource, SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Size: 1},
|
{SourcePath: "report.md", DestinationPath: "report.md", Kind: state.OutputKindSource, SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Size: 1},
|
||||||
{SourcePath: "report.md", DestinationPath: "report.html", Kind: state.OutputKindGenerated, Transform: transform.MarkdownToHTML, URL: "https://reports.example.com/report.html", SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Size: 2},
|
{SourcePath: "report.md", DestinationPath: "report.html", Kind: state.OutputKindGenerated, Transform: transform.MarkdownToHTML, URL: "https://reports.example.com/report.html", SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Size: 2},
|
||||||
}
|
}
|
||||||
|
|
||||||
stateOutputs := StateOutputFiles(outputs)
|
|
||||||
if len(stateOutputs) != 2 || stateOutputs[1].Path != "report.html" || stateOutputs[1].Transform != transform.MarkdownToHTML || stateOutputs[1].URL != outputs[1].URL {
|
|
||||||
t.Fatalf("state outputs = %#v", stateOutputs)
|
|
||||||
}
|
|
||||||
|
|
||||||
paths := ManagedOutputPaths(outputs)
|
paths := ManagedOutputPaths(outputs)
|
||||||
if len(paths) != 2 || paths[0] != "report.md" || paths[1] != "report.html" {
|
if len(paths) != 2 || paths[0] != "report.md" || paths[1] != "report.html" {
|
||||||
t.Fatalf("managed paths = %#v", paths)
|
t.Fatalf("managed paths = %#v", paths)
|
||||||
|
|||||||
@@ -16,15 +16,10 @@ type Action string
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
ActionPublishNew Action = "publish_new"
|
ActionPublishNew Action = "publish_new"
|
||||||
ActionReplaceOlder Action = "replace_older"
|
|
||||||
ActionReplaceConflict Action = "replace_conflict"
|
|
||||||
ActionReplaceNewer Action = "replace_newer"
|
|
||||||
ActionSkipSame Action = "skip_same"
|
ActionSkipSame Action = "skip_same"
|
||||||
ActionSkipDestinationNewer Action = "skip_destination_newer"
|
|
||||||
ActionFailConflict Action = "fail_conflict"
|
ActionFailConflict Action = "fail_conflict"
|
||||||
ActionFailUnmanaged Action = "fail_unmanaged"
|
ActionFailUnmanaged Action = "fail_unmanaged"
|
||||||
ActionForceReplace Action = "force_replace"
|
ActionForceReplace Action = "force_replace"
|
||||||
ActionReplaceTakeover Action = "replace_takeover"
|
|
||||||
ActionUpsertAdditive Action = "upsert_additive"
|
ActionUpsertAdditive Action = "upsert_additive"
|
||||||
ActionReplaceCatalog Action = "replace_catalog"
|
ActionReplaceCatalog Action = "replace_catalog"
|
||||||
)
|
)
|
||||||
@@ -82,18 +77,6 @@ type Plan struct {
|
|||||||
CatalogOutputsToRetain []state.CatalogOutputFile
|
CatalogOutputsToRetain []state.CatalogOutputFile
|
||||||
CatalogOutputsToDelete []state.CatalogOutputFile
|
CatalogOutputsToDelete []state.CatalogOutputFile
|
||||||
ClearDestinationRoot bool
|
ClearDestinationRoot bool
|
||||||
|
|
||||||
// Retained while the executor is migrated to catalog state.
|
|
||||||
StateMode string
|
|
||||||
Reconciliation config.ReconciliationPolicy
|
|
||||||
TakeoverMode string
|
|
||||||
ExistingState *state.DistributorState
|
|
||||||
ExistingSharedRoot *state.SharedRootState
|
|
||||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
|
||||||
TakenOverOwnerOutputs []state.SharedRootOutputFile
|
|
||||||
RetainedOwnerOutputs []state.SharedRootOutputFile
|
|
||||||
OwnerOutputsToDelete []state.SharedRootOutputFile
|
|
||||||
OwnerOutputsToWrite []Output
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type catalogPlanDetails struct {
|
type catalogPlanDetails struct {
|
||||||
@@ -139,8 +122,15 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
|||||||
SupersededLegacy: status.SupersededLegacy,
|
SupersededLegacy: status.SupersededLegacy,
|
||||||
}
|
}
|
||||||
if status.StateErr != nil {
|
if status.StateErr != nil {
|
||||||
plan.Action = ActionFailConflict
|
|
||||||
plan.Reason = status.StateErr.Error()
|
plan.Reason = status.StateErr.Error()
|
||||||
|
if req.Force {
|
||||||
|
plan.Action = ActionForceReplace
|
||||||
|
plan.Force = true
|
||||||
|
plan.ClearDestinationRoot = true
|
||||||
|
plan.CatalogOutputsToWrite = catalogOutputsForPlan(req, outputs, nil, scope, now)
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
plan.Action = ActionFailConflict
|
||||||
return plan, fmt.Errorf("%s: %s", plan.Action, plan.Reason)
|
return plan, fmt.Errorf("%s: %s", plan.Action, plan.Reason)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -151,7 +141,7 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
|||||||
case status.SupersededLegacy != nil:
|
case status.SupersededLegacy != nil:
|
||||||
details = planSupersededLegacy(req, outputs, workflow, scope, now)
|
details = planSupersededLegacy(req, outputs, workflow, scope, now)
|
||||||
default:
|
default:
|
||||||
details, err = planWithoutCatalog(ctx, req, outputs, workflow, scope, now)
|
details, err = planWithoutCatalog(ctx, req, outputs, workflow, scope, now, status.HasContents)
|
||||||
}
|
}
|
||||||
plan.Action = details.Action
|
plan.Action = details.Action
|
||||||
plan.Reason = details.Reason
|
plan.Reason = details.Reason
|
||||||
@@ -159,6 +149,15 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
|||||||
plan.CatalogOutputsToRetain = details.CatalogOutputsToRetain
|
plan.CatalogOutputsToRetain = details.CatalogOutputsToRetain
|
||||||
plan.CatalogOutputsToDelete = details.CatalogOutputsToDelete
|
plan.CatalogOutputsToDelete = details.CatalogOutputsToDelete
|
||||||
plan.ClearDestinationRoot = details.ClearDestinationRoot
|
plan.ClearDestinationRoot = details.ClearDestinationRoot
|
||||||
|
if err != nil && req.Force && forceCanReplace(details.Action) {
|
||||||
|
plan.Action = ActionForceReplace
|
||||||
|
plan.Force = true
|
||||||
|
plan.ClearDestinationRoot = true
|
||||||
|
plan.CatalogOutputsToWrite = catalogOutputsForPlan(req, outputs, nil, scope, now)
|
||||||
|
plan.CatalogOutputsToRetain = nil
|
||||||
|
plan.CatalogOutputsToDelete = nil
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return plan, err
|
return plan, err
|
||||||
}
|
}
|
||||||
@@ -215,6 +214,7 @@ func planExistingCatalog(ctx context.Context, req Request, catalog state.Catalog
|
|||||||
Action: actionForWorkflow(workflow),
|
Action: actionForWorkflow(workflow),
|
||||||
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, catalog.Outputs, scope, now),
|
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, catalog.Outputs, scope, now),
|
||||||
}
|
}
|
||||||
|
allPlannedOutputsMatch := catalogContainsMatchingOutputs(req, catalog.Outputs, outputs, scope)
|
||||||
for _, output := range catalog.Outputs {
|
for _, output := range catalog.Outputs {
|
||||||
if _, exists := planned[output.Path]; exists {
|
if _, exists := planned[output.Path]; exists {
|
||||||
continue
|
continue
|
||||||
@@ -225,9 +225,43 @@ func planExistingCatalog(ctx context.Context, req Request, catalog state.Catalog
|
|||||||
}
|
}
|
||||||
details.CatalogOutputsToRetain = append(details.CatalogOutputsToRetain, output)
|
details.CatalogOutputsToRetain = append(details.CatalogOutputsToRetain, output)
|
||||||
}
|
}
|
||||||
|
if allPlannedOutputsMatch && (workflow == config.WorkflowAdditive || len(details.CatalogOutputsToDelete) == 0) {
|
||||||
|
details.Action = ActionSkipSame
|
||||||
|
details.CatalogOutputsToWrite = nil
|
||||||
|
details.CatalogOutputsToDelete = nil
|
||||||
|
}
|
||||||
return details, nil
|
return details, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func catalogContainsMatchingOutputs(req Request, existing []state.CatalogOutputFile, outputs []Output, scope state.OwnerScope) bool {
|
||||||
|
for _, output := range outputs {
|
||||||
|
catalogOutput, ok := state.FindCatalogOutputByPath(existing, output.DestinationPath)
|
||||||
|
if !ok || !catalogOutputMatchesPlan(req, catalogOutput, output, scope) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func catalogOutputMatchesPlan(req Request, catalogOutput state.CatalogOutputFile, output Output, scope state.OwnerScope) bool {
|
||||||
|
if catalogOutput.PipelineID != scope.PipelineID ||
|
||||||
|
catalogOutput.DestinationID != scope.DestinationID ||
|
||||||
|
catalogOutput.Source.ID != req.SourceBundle.Manifest.ID ||
|
||||||
|
catalogOutput.Source.Digest != req.SourceBundle.Manifest.Digest ||
|
||||||
|
!catalogOutput.Source.Created.Equal(req.SourceBundle.Manifest.Created) ||
|
||||||
|
catalogOutput.Path != output.DestinationPath ||
|
||||||
|
catalogOutput.Kind != output.Kind ||
|
||||||
|
catalogOutput.URL != output.URL ||
|
||||||
|
catalogOutput.SHA256 != output.SHA256 ||
|
||||||
|
catalogOutput.Size != output.Size {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if output.Kind == state.OutputKindGenerated {
|
||||||
|
return catalogOutput.SourcePath == output.SourcePath && catalogOutput.Transform == output.Transform
|
||||||
|
}
|
||||||
|
return catalogOutput.SourcePath == "" && catalogOutput.Transform == ""
|
||||||
|
}
|
||||||
|
|
||||||
func planSupersededLegacy(req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) catalogPlanDetails {
|
func planSupersededLegacy(req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) catalogPlanDetails {
|
||||||
details := catalogPlanDetails{
|
details := catalogPlanDetails{
|
||||||
Action: actionForWorkflow(workflow),
|
Action: actionForWorkflow(workflow),
|
||||||
@@ -239,7 +273,14 @@ func planSupersededLegacy(req Request, outputs []Output, workflow string, scope
|
|||||||
return details
|
return details
|
||||||
}
|
}
|
||||||
|
|
||||||
func planWithoutCatalog(ctx context.Context, req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) (catalogPlanDetails, error) {
|
func planWithoutCatalog(ctx context.Context, req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time, hasContents bool) (catalogPlanDetails, error) {
|
||||||
|
if hasContents {
|
||||||
|
err := fmt.Errorf("destination has content but no distributor state")
|
||||||
|
return catalogPlanDetails{
|
||||||
|
Action: ActionFailUnmanaged,
|
||||||
|
Reason: err.Error(),
|
||||||
|
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||||
|
}
|
||||||
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, nil, outputs); err != nil {
|
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, nil, outputs); err != nil {
|
||||||
return catalogPlanDetails{
|
return catalogPlanDetails{
|
||||||
Action: ActionFailUnmanaged,
|
Action: ActionFailUnmanaged,
|
||||||
@@ -252,6 +293,10 @@ func planWithoutCatalog(ctx context.Context, req Request, outputs []Output, work
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func forceCanReplace(action Action) bool {
|
||||||
|
return action == ActionFailUnmanaged || action == ActionFailConflict
|
||||||
|
}
|
||||||
|
|
||||||
func actionForWorkflow(workflow string) Action {
|
func actionForWorkflow(workflow string) Action {
|
||||||
if workflow == config.WorkflowReplacement {
|
if workflow == config.WorkflowReplacement {
|
||||||
return ActionReplaceCatalog
|
return ActionReplaceCatalog
|
||||||
@@ -334,22 +379,3 @@ func catalogOutputPathSet(outputs []state.CatalogOutputFile) map[string]struct{}
|
|||||||
}
|
}
|
||||||
return paths
|
return paths
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
|
|
||||||
if policy.Mode == "" {
|
|
||||||
policy.Mode = config.ReconciliationModeReplace
|
|
||||||
}
|
|
||||||
return policy
|
|
||||||
}
|
|
||||||
|
|
||||||
func isReconciliationReplacementAction(action Action) bool {
|
|
||||||
return action == ActionReplaceOlder || action == ActionReplaceNewer
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootOutputPathSet(outputs []state.SharedRootOutputFile) map[string]struct{} {
|
|
||||||
paths := make(map[string]struct{}, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
paths[output.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -7,11 +7,13 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -108,6 +110,189 @@ func TestBuildReplacementDeletesCurrentOwnerAndRetainsOtherOwners(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildAdditiveSkipsMatchingCatalogOutputs(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
existing := matchingCatalogForRequest(t, req)
|
||||||
|
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt))
|
||||||
|
writeCatalogState(t, destinationBackend, "", existing)
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != ActionSkipSame {
|
||||||
|
t.Fatalf("plan action = %s, want %s", plan.Action, ActionSkipSame)
|
||||||
|
}
|
||||||
|
if len(plan.CatalogOutputsToWrite) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||||
|
t.Fatalf("catalog write=%d delete=%d, want no writes or deletes", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToDelete))
|
||||||
|
}
|
||||||
|
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "old.txt" {
|
||||||
|
t.Fatalf("retained outputs = %#v, want old.txt", plan.CatalogOutputsToRetain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildReplacementSkipsMatchingCatalogOutputs(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||||
|
existing := matchingCatalogForRequest(t, req)
|
||||||
|
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt))
|
||||||
|
writeCatalogState(t, destinationBackend, "", existing)
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != ActionSkipSame {
|
||||||
|
t.Fatalf("plan action = %s, want %s", plan.Action, ActionSkipSame)
|
||||||
|
}
|
||||||
|
if len(plan.CatalogOutputsToWrite) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||||
|
t.Fatalf("catalog write=%d delete=%d, want no writes or deletes", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToDelete))
|
||||||
|
}
|
||||||
|
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "shared.txt" {
|
||||||
|
t.Fatalf("retained outputs = %#v, want shared.txt", plan.CatalogOutputsToRetain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildReplacementDoesNotSkipWhenCurrentOwnerOutputWouldBeDeleted(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||||
|
existing := matchingCatalogForRequest(t, req)
|
||||||
|
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt))
|
||||||
|
writeCatalogState(t, destinationBackend, "", existing)
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != ActionReplaceCatalog {
|
||||||
|
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceCatalog)
|
||||||
|
}
|
||||||
|
if len(plan.CatalogOutputsToDelete) != 1 || plan.CatalogOutputsToDelete[0].Path != "stale.txt" {
|
||||||
|
t.Fatalf("delete outputs = %#v, want stale.txt", plan.CatalogOutputsToDelete)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildDoesNotSkipWhenCatalogMetadataDiffers(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
configure func(*Request)
|
||||||
|
mutate func(*state.CatalogState)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "pipeline id",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].PipelineID = "other"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "destination id",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].DestinationID = "web"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "source id",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].Source.ID = "reports.other"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "source digest",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].Source.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "source created",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].Source.Created = catalog.Outputs[0].Source.Created.Add(time.Second)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "output kind",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].Kind = state.OutputKindGenerated
|
||||||
|
catalog.Outputs[0].SourcePath = "report.md"
|
||||||
|
catalog.Outputs[0].Transform = transform.MarkdownToHTML
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "output digest",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].SHA256 = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "output size",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].Size++
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "url",
|
||||||
|
configure: func(req *Request) {
|
||||||
|
req.Links = &config.Links{BaseURL: "https://reports.example.com/archive", Primary: config.LinkPrimaryAuto}
|
||||||
|
},
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].URL = "https://reports.example.com/archive/old-report.md"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
if tt.configure != nil {
|
||||||
|
tt.configure(&req)
|
||||||
|
}
|
||||||
|
existing := matchingCatalogForRequest(t, req)
|
||||||
|
tt.mutate(&existing)
|
||||||
|
writeCatalogState(t, destinationBackend, "", existing)
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action == ActionSkipSame {
|
||||||
|
t.Fatalf("plan action = %s, want write action after metadata change", plan.Action)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildDoesNotSkipWhenGeneratedCatalogMetadataDiffers(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*state.CatalogState)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "source path",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].SourcePath = "summary.md"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "transform",
|
||||||
|
mutate: func(catalog *state.CatalogState) {
|
||||||
|
catalog.Outputs[0].Transform = "markdown_to_html_index"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogGeneratedPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
existing := matchingCatalogForRequest(t, req)
|
||||||
|
tt.mutate(&existing)
|
||||||
|
writeCatalogState(t, destinationBackend, "", existing)
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action == ActionSkipSame {
|
||||||
|
t.Fatalf("plan action = %s, want write action after generated metadata change", plan.Action)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildTransfersManagedPathOwnership(t *testing.T) {
|
func TestBuildTransfersManagedPathOwnership(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||||
existing := baseCatalog(req)
|
existing := baseCatalog(req)
|
||||||
@@ -137,6 +322,7 @@ func TestBuildTransfersManagedPathOwnership(t *testing.T) {
|
|||||||
|
|
||||||
func TestBuildRejectsUnmanagedPlannedPathCollision(t *testing.T) {
|
func TestBuildRejectsUnmanagedPlannedPathCollision(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
writeCatalogState(t, destinationBackend, "", baseCatalog(req))
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||||
|
|
||||||
plan, err := Build(context.Background(), req)
|
plan, err := Build(context.Background(), req)
|
||||||
@@ -148,9 +334,58 @@ func TestBuildRejectsUnmanagedPlannedPathCollision(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildForceReplacesUnmanagedPlannedPathCollision(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
req.Force = true
|
||||||
|
existing := baseCatalog(req)
|
||||||
|
writeCatalogState(t, destinationBackend, "", existing)
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
|
||||||
|
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
|
||||||
|
}
|
||||||
|
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||||
|
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildRejectsNoStateNonEmptyDestinationWithoutForce(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "unmanaged")
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Build() error = nil, want unmanaged destination")
|
||||||
|
}
|
||||||
|
if plan.Action != ActionFailUnmanaged || !strings.Contains(err.Error(), "destination has content but no distributor state") {
|
||||||
|
t.Fatalf("plan action=%s error=%v, want unmanaged destination", plan.Action, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildForceReplacesNoStateNonEmptyDestination(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
req.Force = true
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "unmanaged")
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
|
||||||
|
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
|
||||||
|
}
|
||||||
|
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||||
|
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
legacyState := legacyStateDocument(2)
|
||||||
writeJSONState(t, destinationBackend, "", legacyState)
|
writeJSONState(t, destinationBackend, "", legacyState)
|
||||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
|
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
|
||||||
|
|
||||||
@@ -158,7 +393,7 @@ func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Build() error = %v", err)
|
t.Fatalf("Build() error = %v", err)
|
||||||
}
|
}
|
||||||
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != state.SchemaVersion {
|
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != 2 {
|
||||||
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
|
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
|
||||||
}
|
}
|
||||||
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
|
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
|
||||||
@@ -168,7 +403,7 @@ func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
|
|||||||
|
|
||||||
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
|
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
|
||||||
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
|
||||||
legacyState := testutil.DestinationState(req.SourceBundle.Manifest, testutil.DestinationStateOptions{})
|
legacyState := legacyStateDocument(2)
|
||||||
writeJSONState(t, destinationBackend, "", legacyState)
|
writeJSONState(t, destinationBackend, "", legacyState)
|
||||||
|
|
||||||
plan, err := Build(context.Background(), req)
|
plan, err := Build(context.Background(), req)
|
||||||
@@ -204,6 +439,60 @@ func TestBuildRejectsInvalidOrFutureState(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildForceReplacesInvalidOrFutureState(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
data string
|
||||||
|
}{
|
||||||
|
{name: "invalid json", data: `{"schema_version":`},
|
||||||
|
{name: "future schema", data: `{"schema_version":99}`},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
|
||||||
|
req.Force = true
|
||||||
|
testutil.WriteFakeFile(t, destinationBackend, storage.StateFileName, tt.data)
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
|
||||||
|
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
|
||||||
|
}
|
||||||
|
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
|
||||||
|
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildForceDoesNotChangeValidCatalogActions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
workflow string
|
||||||
|
want Action
|
||||||
|
}{
|
||||||
|
{name: "additive", workflow: config.WorkflowAdditive, want: ActionUpsertAdditive},
|
||||||
|
{name: "replacement", workflow: config.WorkflowReplacement, want: ActionReplaceCatalog},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, destinationBackend, req := catalogPlanRequest(t, tt.workflow)
|
||||||
|
req.Force = true
|
||||||
|
writeCatalogState(t, destinationBackend, "", baseCatalog(req))
|
||||||
|
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.Action != tt.want || plan.Force || plan.ClearDestinationRoot {
|
||||||
|
t.Fatalf("plan action=%s force=%t clear=%t, want %s without forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateRequestRejectsInvalidWorkflow(t *testing.T) {
|
func TestValidateRequestRejectsInvalidWorkflow(t *testing.T) {
|
||||||
sourceBackend, destinationBackend, req := catalogPlanRequest(t, "append")
|
sourceBackend, destinationBackend, req := catalogPlanRequest(t, "append")
|
||||||
req.SourceBackend = sourceBackend
|
req.SourceBackend = sourceBackend
|
||||||
@@ -238,6 +527,41 @@ func catalogPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Bac
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func catalogGeneratedPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Backend, Request) {
|
||||||
|
t.Helper()
|
||||||
|
sourceBackend, destinationBackend, req := catalogPlanRequest(t, workflow)
|
||||||
|
data := []byte("<p>Generated</p>\n")
|
||||||
|
req.Publish = config.PublishPolicy{HTML: true}
|
||||||
|
req.Transform = config.Transform{MarkdownToHTML: &config.MarkdownToHTML{
|
||||||
|
Enabled: true,
|
||||||
|
Mode: config.TransformModeSidecar,
|
||||||
|
}}
|
||||||
|
req.Transformers = testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
|
||||||
|
Path: "report.html",
|
||||||
|
SourcePath: "report.md",
|
||||||
|
Transform: transform.MarkdownToHTML,
|
||||||
|
Data: data,
|
||||||
|
SHA256: bundle.FileDigest(data),
|
||||||
|
Size: int64(len(data)),
|
||||||
|
}}}}
|
||||||
|
return sourceBackend, destinationBackend, req
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchingCatalogForRequest(t *testing.T, req Request) state.CatalogState {
|
||||||
|
t.Helper()
|
||||||
|
outputs, err := PlanOutputs(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanOutputs() error = %v", err)
|
||||||
|
}
|
||||||
|
outputs, _, err = PlanLinks(req, outputs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PlanLinks() error = %v", err)
|
||||||
|
}
|
||||||
|
catalog := baseCatalog(req)
|
||||||
|
catalog.Outputs = catalogOutputsForPlan(req, outputs, nil, state.CurrentOwnerScope(req.PipelineID, req.DestinationID), planCreatedAt)
|
||||||
|
return catalog
|
||||||
|
}
|
||||||
|
|
||||||
func baseCatalog(req Request) state.CatalogState {
|
func baseCatalog(req Request) state.CatalogState {
|
||||||
return state.CatalogState{
|
return state.CatalogState{
|
||||||
SchemaVersion: state.CatalogSchemaVersion,
|
SchemaVersion: state.CatalogSchemaVersion,
|
||||||
@@ -282,6 +606,10 @@ func writeCatalogState(t *testing.T, backend *fake.Backend, relative string, cat
|
|||||||
writeJSONState(t, backend, relative, catalog)
|
writeJSONState(t, backend, relative, catalog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func legacyStateDocument(schemaVersion int) map[string]int {
|
||||||
|
return map[string]int{"schema_version": schemaVersion}
|
||||||
|
}
|
||||||
|
|
||||||
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
|
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := json.MarshalIndent(value, "", " ")
|
data, err := json.MarshalIndent(value, "", " ")
|
||||||
|
|||||||
@@ -19,8 +19,6 @@ func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath
|
|||||||
return state.DestinationStatus{StateErr: parseErr}, nil
|
return state.DestinationStatus{StateErr: parseErr}, nil
|
||||||
}
|
}
|
||||||
return state.DestinationStatus{
|
return state.DestinationStatus{
|
||||||
State: document.SingleOwner,
|
|
||||||
SharedRoot: document.SharedRoot,
|
|
||||||
Catalog: document.Catalog,
|
Catalog: document.Catalog,
|
||||||
SupersededLegacy: document.SupersededLegacy,
|
SupersededLegacy: document.SupersededLegacy,
|
||||||
HasContents: true,
|
HasContents: true,
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseCatalogState(t *testing.T) {
|
func TestParseCatalogState(t *testing.T) {
|
||||||
@@ -45,7 +47,7 @@ func TestCatalogMarshalIsDeterministic(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
t.Fatalf("Marshal() error = %v", err)
|
||||||
}
|
}
|
||||||
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
|
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
|
||||||
if string(data) != want {
|
if string(data) != want {
|
||||||
t.Fatalf("json = %s, want %s", data, want)
|
t.Fatalf("json = %s, want %s", data, want)
|
||||||
}
|
}
|
||||||
@@ -56,14 +58,14 @@ func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ParseDocument(catalog) error = %v", err)
|
t.Fatalf("ParseDocument(catalog) error = %v", err)
|
||||||
}
|
}
|
||||||
if catalog.Catalog == nil || catalog.SingleOwner != nil || catalog.SharedRoot != nil || catalog.SupersededLegacy != nil {
|
if catalog.Catalog == nil || catalog.SupersededLegacy != nil {
|
||||||
t.Fatalf("catalog document = %#v", catalog)
|
t.Fatalf("catalog document = %#v", catalog)
|
||||||
}
|
}
|
||||||
|
|
||||||
tests := map[string]string{
|
tests := map[string]string{
|
||||||
"schema 1": legacyStateJSON(t),
|
"schema 1": schemaOnlyStateJSON(legacySchemaVersion),
|
||||||
"schema 2": validStateJSON(t),
|
"schema 2": schemaOnlyStateJSON(legacyOwnerSchema),
|
||||||
"schema 3": validSharedRootStateJSON(t),
|
"schema 3": schemaOnlyStateJSON(legacyMultiOwnerSchema),
|
||||||
}
|
}
|
||||||
for name, body := range tests {
|
for name, body := range tests {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
@@ -71,7 +73,7 @@ func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ParseDocument() error = %v", err)
|
t.Fatalf("ParseDocument() error = %v", err)
|
||||||
}
|
}
|
||||||
if document.SupersededLegacy == nil || document.Catalog != nil || document.SingleOwner != nil || document.SharedRoot != nil {
|
if document.SupersededLegacy == nil || document.Catalog != nil {
|
||||||
t.Fatalf("document = %#v, want superseded legacy only", document)
|
t.Fatalf("document = %#v, want superseded legacy only", document)
|
||||||
}
|
}
|
||||||
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
|
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
|
||||||
@@ -317,6 +319,41 @@ func mustMarshalCatalogObject(t *testing.T, document map[string]any) []byte {
|
|||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func schemaOnlyStateJSON(schemaVersion int) string {
|
||||||
|
data, err := json.Marshal(map[string]int{"schema_version": schemaVersion})
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStateErrorContains(t *testing.T, err error, substring string) {
|
||||||
|
t.Helper()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("error = nil, want substring %q", substring)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), substring) {
|
||||||
|
t.Fatalf("error = %v, want substring %q", err, substring)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validManifest(t *testing.T) bundle.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
files := []bundle.ManifestFile{{
|
||||||
|
Path: "report.md",
|
||||||
|
SHA256: bundle.FileDigest([]byte("# Report\nSunny.\n")),
|
||||||
|
Size: int64(len("# Report\nSunny.\n")),
|
||||||
|
}}
|
||||||
|
manifest := bundle.Manifest{
|
||||||
|
SchemaVersion: bundle.SchemaVersion,
|
||||||
|
ID: "weather.daily.brentwood.2026-05-30",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: files,
|
||||||
|
}
|
||||||
|
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||||
|
return manifest
|
||||||
|
}
|
||||||
|
|
||||||
func validCatalogState(t *testing.T) CatalogState {
|
func validCatalogState(t *testing.T) CatalogState {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
manifest := validManifest(t)
|
manifest := validManifest(t)
|
||||||
|
|||||||
@@ -1,216 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Outcome string
|
|
||||||
|
|
||||||
const (
|
|
||||||
OutcomeDestinationAbsent Outcome = "destination_absent"
|
|
||||||
OutcomeDestinationUnmanaged Outcome = "destination_unmanaged"
|
|
||||||
OutcomeInvalidState Outcome = "invalid_destination_state"
|
|
||||||
OutcomeIdentityMismatch Outcome = "destination_identity_mismatch"
|
|
||||||
OutcomeSameSource Outcome = "same_source_manifest"
|
|
||||||
OutcomeDestinationOlder Outcome = "destination_older"
|
|
||||||
OutcomeDestinationNewer Outcome = "destination_newer"
|
|
||||||
OutcomeSameCreatedConflict Outcome = "same_created_digest_conflict"
|
|
||||||
OutcomeDifferentSourceConflict Outcome = "different_source_conflict"
|
|
||||||
)
|
|
||||||
|
|
||||||
type DestinationStatus struct {
|
|
||||||
State *DistributorState
|
|
||||||
SharedRoot *SharedRootState
|
|
||||||
Catalog *CatalogState
|
|
||||||
SupersededLegacy *SupersededLegacyState
|
|
||||||
StateErr error
|
|
||||||
HasContents bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type Comparison struct {
|
|
||||||
Outcome Outcome
|
|
||||||
Reason string
|
|
||||||
Detail ComparisonDetail
|
|
||||||
}
|
|
||||||
|
|
||||||
type ComparisonDetailKind string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ComparisonDetailNone ComparisonDetailKind = ""
|
|
||||||
ComparisonDetailInvalidState ComparisonDetailKind = "invalid_state"
|
|
||||||
ComparisonDetailUnmanagedContent ComparisonDetailKind = "unmanaged_content"
|
|
||||||
ComparisonDetailPipelineIDMismatch ComparisonDetailKind = "pipeline_id_mismatch"
|
|
||||||
ComparisonDetailDestinationIDMismatch ComparisonDetailKind = "destination_id_mismatch"
|
|
||||||
ComparisonDetailDifferentSourceID ComparisonDetailKind = "different_source_id"
|
|
||||||
ComparisonDetailSameCreatedDigestConflict ComparisonDetailKind = "same_created_digest_conflict"
|
|
||||||
ComparisonDetailDestinationNewer ComparisonDetailKind = "destination_newer"
|
|
||||||
ComparisonDetailSharedRootOwnerAbsent ComparisonDetailKind = "shared_root_owner_absent"
|
|
||||||
ComparisonDetailSharedRootOutputOwner ComparisonDetailKind = "shared_root_output_owner_conflict"
|
|
||||||
)
|
|
||||||
|
|
||||||
type ComparisonDetail struct {
|
|
||||||
Kind ComparisonDetailKind
|
|
||||||
CurrentPipelineID string
|
|
||||||
CurrentDestinationID string
|
|
||||||
DestinationPipelineID string
|
|
||||||
DestinationDestinationID string
|
|
||||||
CurrentSourceID string
|
|
||||||
DestinationSourceID string
|
|
||||||
CurrentSourceDigest string
|
|
||||||
DestinationSourceDigest string
|
|
||||||
Path string
|
|
||||||
CurrentOwner OwnerScope
|
|
||||||
ConflictingOwner OwnerScope
|
|
||||||
}
|
|
||||||
|
|
||||||
func CompareSharedRootOwner(source bundle.Manifest, scope OwnerScope, status DestinationStatus) Comparison {
|
|
||||||
if status.StateErr != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
if status.SharedRoot != nil {
|
|
||||||
if err := ValidateSharedRoot(*status.SharedRoot); err != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
owner, ok := status.SharedRoot.Owner(scope)
|
|
||||||
if !ok {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeDestinationAbsent,
|
|
||||||
Reason: fmt.Sprintf("destination owner %s/%s is absent", scope.PipelineID, scope.DestinationID),
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailSharedRootOwnerAbsent,
|
|
||||||
CurrentOwner: scope,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return compareManifests(source, owner.Source.Manifest)
|
|
||||||
}
|
|
||||||
if status.State != nil {
|
|
||||||
destinationState := *status.State
|
|
||||||
if err := Validate(destinationState); err != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
if destinationState.PipelineID != scope.PipelineID {
|
|
||||||
return pipelineIDMismatchComparison(destinationState.PipelineID, scope.PipelineID)
|
|
||||||
}
|
|
||||||
if destinationState.DestinationID != scope.DestinationID {
|
|
||||||
return destinationIDMismatchComparison(destinationState.DestinationID, scope.DestinationID)
|
|
||||||
}
|
|
||||||
return compareManifests(source, destinationState.Source.Manifest)
|
|
||||||
}
|
|
||||||
if status.HasContents {
|
|
||||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state", Detail: ComparisonDetail{Kind: ComparisonDetailUnmanagedContent}}
|
|
||||||
}
|
|
||||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
|
||||||
}
|
|
||||||
|
|
||||||
func compareManifests(source, destination bundle.Manifest) Comparison {
|
|
||||||
if manifestsEqual(source, destination) {
|
|
||||||
return Comparison{Outcome: OutcomeSameSource, Reason: "destination source manifest matches source"}
|
|
||||||
}
|
|
||||||
if destination.ID != source.ID {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeDifferentSourceConflict,
|
|
||||||
Reason: "destination source id differs from source",
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailDifferentSourceID,
|
|
||||||
CurrentSourceID: source.ID,
|
|
||||||
DestinationSourceID: destination.ID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if destination.Created.Before(source.Created) {
|
|
||||||
return Comparison{Outcome: OutcomeDestinationOlder, Reason: "destination source is older than source"}
|
|
||||||
}
|
|
||||||
if destination.Created.After(source.Created) {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeDestinationNewer,
|
|
||||||
Reason: "destination source is newer than source",
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailDestinationNewer,
|
|
||||||
CurrentSourceID: source.ID,
|
|
||||||
DestinationSourceID: destination.ID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if destination.Digest != source.Digest {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeSameCreatedConflict,
|
|
||||||
Reason: "destination source has same id and created time but different digest",
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailSameCreatedDigestConflict,
|
|
||||||
CurrentSourceID: source.ID,
|
|
||||||
DestinationSourceID: destination.ID,
|
|
||||||
CurrentSourceDigest: source.Digest,
|
|
||||||
DestinationSourceDigest: destination.Digest,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: "destination source differs from source without a supported comparison outcome", Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Compare(source bundle.Manifest, pipelineID, destinationID string, status DestinationStatus) Comparison {
|
|
||||||
if status.StateErr != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
if status.State == nil {
|
|
||||||
if status.HasContents {
|
|
||||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state", Detail: ComparisonDetail{Kind: ComparisonDetailUnmanagedContent}}
|
|
||||||
}
|
|
||||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
|
||||||
}
|
|
||||||
|
|
||||||
destinationState := *status.State
|
|
||||||
if err := Validate(destinationState); err != nil {
|
|
||||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error(), Detail: ComparisonDetail{Kind: ComparisonDetailInvalidState}}
|
|
||||||
}
|
|
||||||
if destinationState.PipelineID != pipelineID {
|
|
||||||
return pipelineIDMismatchComparison(destinationState.PipelineID, pipelineID)
|
|
||||||
}
|
|
||||||
if destinationState.DestinationID != destinationID {
|
|
||||||
return destinationIDMismatchComparison(destinationState.DestinationID, destinationID)
|
|
||||||
}
|
|
||||||
|
|
||||||
return compareManifests(source, destinationState.Source.Manifest)
|
|
||||||
}
|
|
||||||
|
|
||||||
func pipelineIDMismatchComparison(destinationPipelineID, currentPipelineID string) Comparison {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeIdentityMismatch,
|
|
||||||
Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationPipelineID, currentPipelineID),
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailPipelineIDMismatch,
|
|
||||||
CurrentPipelineID: currentPipelineID,
|
|
||||||
DestinationPipelineID: destinationPipelineID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func destinationIDMismatchComparison(destinationDestinationID, currentDestinationID string) Comparison {
|
|
||||||
return Comparison{
|
|
||||||
Outcome: OutcomeIdentityMismatch,
|
|
||||||
Reason: fmt.Sprintf("destination id %q does not match %q", destinationDestinationID, currentDestinationID),
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailDestinationIDMismatch,
|
|
||||||
CurrentDestinationID: currentDestinationID,
|
|
||||||
DestinationDestinationID: destinationDestinationID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func manifestsEqual(a, b bundle.Manifest) bool {
|
|
||||||
if a.SchemaVersion != b.SchemaVersion ||
|
|
||||||
a.ID != b.ID ||
|
|
||||||
a.Digest != b.Digest ||
|
|
||||||
!a.Created.Equal(b.Created) ||
|
|
||||||
len(a.Files) != len(b.Files) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for index := range a.Files {
|
|
||||||
if a.Files[index] != b.Files[index] {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
@@ -1,216 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCompareOutcomes(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
status DestinationStatus
|
|
||||||
want Outcome
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "destination absent",
|
|
||||||
status: DestinationStatus{},
|
|
||||||
want: OutcomeDestinationAbsent,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination unmanaged",
|
|
||||||
status: DestinationStatus{HasContents: true},
|
|
||||||
want: OutcomeDestinationUnmanaged,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid destination state",
|
|
||||||
status: DestinationStatus{StateErr: errors.New("invalid json")},
|
|
||||||
want: OutcomeInvalidState,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "pipeline mismatch",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
|
|
||||||
want: OutcomeIdentityMismatch,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination mismatch",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
|
|
||||||
want: OutcomeIdentityMismatch,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "same source manifest",
|
|
||||||
status: DestinationStatus{State: withState(t, source, nil)},
|
|
||||||
want: OutcomeSameSource,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination older",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Created = source.Created.Add(-time.Hour)
|
|
||||||
})},
|
|
||||||
want: OutcomeDestinationOlder,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination newer",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Created = source.Created.Add(time.Hour)
|
|
||||||
})},
|
|
||||||
want: OutcomeDestinationNewer,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "same created digest conflict",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
|
||||||
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
|
|
||||||
})},
|
|
||||||
want: OutcomeSameCreatedConflict,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "different source id",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.ID = "other.source"
|
|
||||||
})},
|
|
||||||
want: OutcomeDifferentSourceConflict,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid state object",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Kind = "other"
|
|
||||||
})},
|
|
||||||
want: OutcomeInvalidState,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := Compare(source, "reports", "archive", tt.status)
|
|
||||||
if got.Outcome != tt.want {
|
|
||||||
t.Fatalf("Compare() outcome = %s reason=%q, want %s", got.Outcome, got.Reason, tt.want)
|
|
||||||
}
|
|
||||||
if got.Reason == "" {
|
|
||||||
t.Fatal("Compare() reason is empty")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompareReportsStructuredDetails(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
status DestinationStatus
|
|
||||||
wantKind ComparisonDetailKind
|
|
||||||
assertions func(t *testing.T, detail ComparisonDetail)
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "pipeline mismatch",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
|
|
||||||
wantKind: ComparisonDetailPipelineIDMismatch,
|
|
||||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
|
||||||
t.Helper()
|
|
||||||
if detail.DestinationPipelineID != "other" || detail.CurrentPipelineID != "reports" {
|
|
||||||
t.Fatalf("detail = %#v, want pipeline ids", detail)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination mismatch",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
|
|
||||||
wantKind: ComparisonDetailDestinationIDMismatch,
|
|
||||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
|
||||||
t.Helper()
|
|
||||||
if detail.DestinationDestinationID != "other" || detail.CurrentDestinationID != "archive" {
|
|
||||||
t.Fatalf("detail = %#v, want destination ids", detail)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "different source id",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.Source.Manifest.ID = "other.source" })},
|
|
||||||
wantKind: ComparisonDetailDifferentSourceID,
|
|
||||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
|
||||||
t.Helper()
|
|
||||||
if detail.DestinationSourceID != "other.source" || detail.CurrentSourceID != source.ID {
|
|
||||||
t.Fatalf("detail = %#v, want source ids", detail)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "same created digest conflict",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
|
||||||
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
|
|
||||||
})},
|
|
||||||
wantKind: ComparisonDetailSameCreatedDigestConflict,
|
|
||||||
assertions: func(t *testing.T, detail ComparisonDetail) {
|
|
||||||
t.Helper()
|
|
||||||
if detail.CurrentSourceDigest == "" || detail.DestinationSourceDigest == "" || detail.CurrentSourceDigest == detail.DestinationSourceDigest {
|
|
||||||
t.Fatalf("detail = %#v, want different source digests", detail)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "destination newer",
|
|
||||||
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Source.Manifest.Created = source.Created.Add(time.Hour)
|
|
||||||
})},
|
|
||||||
wantKind: ComparisonDetailDestinationNewer,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid state",
|
|
||||||
status: DestinationStatus{StateErr: errors.New("invalid json")},
|
|
||||||
wantKind: ComparisonDetailInvalidState,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unmanaged content",
|
|
||||||
status: DestinationStatus{HasContents: true},
|
|
||||||
wantKind: ComparisonDetailUnmanagedContent,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := Compare(source, "reports", "archive", tt.status)
|
|
||||||
if got.Detail.Kind != tt.wantKind {
|
|
||||||
t.Fatalf("Compare() detail kind = %q, want %q; comparison=%#v", got.Detail.Kind, tt.wantKind, got)
|
|
||||||
}
|
|
||||||
if tt.assertions != nil {
|
|
||||||
tt.assertions(t, got.Detail)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func withState(t *testing.T, source bundle.Manifest, mutate func(*DistributorState)) *DistributorState {
|
|
||||||
t.Helper()
|
|
||||||
stateManifest := source
|
|
||||||
stateManifest.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
|
||||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
state := DistributorState{
|
|
||||||
SchemaVersion: SchemaVersion,
|
|
||||||
PipelineID: "reports",
|
|
||||||
DestinationID: "archive",
|
|
||||||
PublishedAt: publishedAt,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: SourceState{Manifest: stateManifest},
|
|
||||||
Outputs: []OutputFile{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
SHA256: source.Files[0].SHA256,
|
|
||||||
Size: source.Files[0].Size,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if mutate != nil {
|
|
||||||
mutate(&state)
|
|
||||||
}
|
|
||||||
return &state
|
|
||||||
}
|
|
||||||
@@ -1,373 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
SchemaVersion = 2
|
|
||||||
SharedRootSchemaVersion = 3
|
|
||||||
CatalogSchemaVersion = 4
|
|
||||||
legacySchemaVersion = 1
|
|
||||||
StateModeSingleOwner = config.StateModeSingleOwner
|
|
||||||
StateModeSharedRoot = config.StateModeSharedRoot
|
|
||||||
StateModeCatalog = "catalog"
|
|
||||||
)
|
|
||||||
|
|
||||||
type DistributorState struct {
|
|
||||||
SchemaVersion int
|
|
||||||
DistributorVersion string
|
|
||||||
PipelineID string
|
|
||||||
DestinationID string
|
|
||||||
PublishedAt time.Time
|
|
||||||
CreatedAt time.Time
|
|
||||||
UpdatedAt time.Time
|
|
||||||
State StatePolicy
|
|
||||||
Reconciliation ReconciliationPolicy
|
|
||||||
Source SourceState
|
|
||||||
Links *LinkState
|
|
||||||
Outputs []OutputFile
|
|
||||||
}
|
|
||||||
|
|
||||||
type StatePolicy struct {
|
|
||||||
Mode string
|
|
||||||
}
|
|
||||||
|
|
||||||
type ReconciliationPolicy struct {
|
|
||||||
Mode string
|
|
||||||
}
|
|
||||||
|
|
||||||
type SourceState struct {
|
|
||||||
Manifest bundle.Manifest
|
|
||||||
}
|
|
||||||
|
|
||||||
type LinkState struct {
|
|
||||||
PrimaryURL string
|
|
||||||
}
|
|
||||||
|
|
||||||
type OutputFile struct {
|
|
||||||
Path string
|
|
||||||
Kind string
|
|
||||||
SourcePath string
|
|
||||||
Transform string
|
|
||||||
URL string
|
|
||||||
SHA256 string
|
|
||||||
Size int64
|
|
||||||
CreatedAt time.Time
|
|
||||||
UpdatedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawDistributorState struct {
|
|
||||||
SchemaVersion *int `json:"schema_version"`
|
|
||||||
DistributorVersion string `json:"distributor_version"`
|
|
||||||
PipelineID *string `json:"pipeline_id"`
|
|
||||||
DestinationID *string `json:"destination_id"`
|
|
||||||
PublishedAt *string `json:"published_at"`
|
|
||||||
CreatedAt *string `json:"created_at"`
|
|
||||||
UpdatedAt *string `json:"updated_at"`
|
|
||||||
State *rawStatePolicy `json:"state"`
|
|
||||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
|
||||||
Source *rawSourceState `json:"source"`
|
|
||||||
Links *rawLinkState `json:"links"`
|
|
||||||
Outputs []rawOutputFile `json:"outputs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawStatePolicy struct {
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawReconciliationPolicy struct {
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawSourceState struct {
|
|
||||||
Manifest json.RawMessage `json:"manifest"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawLinkState struct {
|
|
||||||
PrimaryURL string `json:"primary_url"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawOutputFile struct {
|
|
||||||
Path *string `json:"path"`
|
|
||||||
Kind *string `json:"kind"`
|
|
||||||
SourcePath *string `json:"source_path"`
|
|
||||||
Transform string `json:"transform"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
SHA256 *string `json:"sha256"`
|
|
||||||
Size *int64 `json:"size"`
|
|
||||||
CreatedAt *string `json:"created_at"`
|
|
||||||
UpdatedAt *string `json:"updated_at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func Parse(data []byte) (DistributorState, error) {
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
||||||
var raw rawDistributorState
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("parse distributor state: %w", err)
|
|
||||||
}
|
|
||||||
var extra any
|
|
||||||
if err := decoder.Decode(&extra); err != io.EOF {
|
|
||||||
return DistributorState{}, fmt.Errorf("parse distributor state: trailing data")
|
|
||||||
}
|
|
||||||
state, err := parseRaw(raw)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
if err := Validate(state); err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
|
||||||
var state DistributorState
|
|
||||||
if raw.SchemaVersion == nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("state schema_version is required")
|
|
||||||
}
|
|
||||||
state.SchemaVersion = *raw.SchemaVersion
|
|
||||||
if state.SchemaVersion != SchemaVersion && state.SchemaVersion != legacySchemaVersion {
|
|
||||||
return DistributorState{}, fmt.Errorf("state schema_version must be %d or %d", legacySchemaVersion, SchemaVersion)
|
|
||||||
}
|
|
||||||
legacy := state.SchemaVersion == legacySchemaVersion
|
|
||||||
state.DistributorVersion = raw.DistributorVersion
|
|
||||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state pipeline_id is required")
|
|
||||||
}
|
|
||||||
state.PipelineID = *raw.PipelineID
|
|
||||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state destination_id is required")
|
|
||||||
}
|
|
||||||
state.DestinationID = *raw.DestinationID
|
|
||||||
if raw.PublishedAt == nil || *raw.PublishedAt == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state published_at is required")
|
|
||||||
}
|
|
||||||
publishedAt, err := time.Parse(time.RFC3339, *raw.PublishedAt)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("state published_at must be RFC3339: %w", err)
|
|
||||||
}
|
|
||||||
state.PublishedAt = publishedAt.UTC()
|
|
||||||
if legacy {
|
|
||||||
state.SchemaVersion = SchemaVersion
|
|
||||||
state.CreatedAt = state.PublishedAt
|
|
||||||
state.UpdatedAt = state.PublishedAt
|
|
||||||
state.State.Mode = StateModeSingleOwner
|
|
||||||
state.Reconciliation.Mode = config.ReconciliationModeReplace
|
|
||||||
} else {
|
|
||||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
state.CreatedAt = createdAt
|
|
||||||
state.UpdatedAt = updatedAt
|
|
||||||
if raw.State == nil || raw.State.Mode == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state state.mode is required")
|
|
||||||
}
|
|
||||||
state.State.Mode = raw.State.Mode
|
|
||||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
|
||||||
return DistributorState{}, fmt.Errorf("state reconciliation.mode is required")
|
|
||||||
}
|
|
||||||
state.Reconciliation.Mode = raw.Reconciliation.Mode
|
|
||||||
}
|
|
||||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
|
||||||
return DistributorState{}, fmt.Errorf("state source.manifest is required")
|
|
||||||
}
|
|
||||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
|
|
||||||
}
|
|
||||||
state.Source.Manifest = manifest
|
|
||||||
if raw.Links != nil {
|
|
||||||
state.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
|
||||||
}
|
|
||||||
if raw.Outputs == nil {
|
|
||||||
return DistributorState{}, fmt.Errorf("state outputs is required")
|
|
||||||
}
|
|
||||||
outputs, err := parseOutputs(raw.Outputs, legacy, state.PublishedAt)
|
|
||||||
if err != nil {
|
|
||||||
return DistributorState{}, err
|
|
||||||
}
|
|
||||||
state.Outputs = outputs
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseRequiredTime(context string, raw *string) (time.Time, error) {
|
|
||||||
if raw == nil || *raw == "" {
|
|
||||||
return time.Time{}, fmt.Errorf("%s is required", context)
|
|
||||||
}
|
|
||||||
parsed, err := time.Parse(time.RFC3339, *raw)
|
|
||||||
if err != nil {
|
|
||||||
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", context, err)
|
|
||||||
}
|
|
||||||
return parsed.UTC(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOutputs(rawOutputs []rawOutputFile, legacy bool, publishedAt time.Time) ([]OutputFile, error) {
|
|
||||||
outputs := make([]OutputFile, 0, len(rawOutputs))
|
|
||||||
seen := make(map[string]struct{}, len(rawOutputs))
|
|
||||||
for index, raw := range rawOutputs {
|
|
||||||
output, err := parseOutput(index, raw, legacy, publishedAt)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if _, exists := seen[output.Path]; exists {
|
|
||||||
return nil, fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
|
||||||
}
|
|
||||||
seen[output.Path] = struct{}{}
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
return outputs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOutput(index int, raw rawOutputFile, legacy bool, publishedAt time.Time) (OutputFile, error) {
|
|
||||||
if raw.Path == nil || *raw.Path == "" {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
|
||||||
}
|
|
||||||
if raw.Kind == nil || *raw.Kind == "" {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
|
||||||
}
|
|
||||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
|
||||||
}
|
|
||||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
|
||||||
}
|
|
||||||
if raw.Size == nil {
|
|
||||||
return OutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
|
||||||
}
|
|
||||||
createdAt := publishedAt
|
|
||||||
updatedAt := publishedAt
|
|
||||||
if !legacy {
|
|
||||||
var err error
|
|
||||||
createdAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return OutputFile{}, err
|
|
||||||
}
|
|
||||||
updatedAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return OutputFile{}, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return OutputFile{
|
|
||||||
Path: *raw.Path,
|
|
||||||
Kind: *raw.Kind,
|
|
||||||
SourcePath: *raw.SourcePath,
|
|
||||||
Transform: raw.Transform,
|
|
||||||
URL: raw.URL,
|
|
||||||
SHA256: *raw.SHA256,
|
|
||||||
Size: *raw.Size,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s DistributorState) PublishedAtString() string {
|
|
||||||
return s.PublishedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s DistributorState) CreatedAtString() string {
|
|
||||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s DistributorState) UpdatedAtString() string {
|
|
||||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o OutputFile) CreatedAtString() string {
|
|
||||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o OutputFile) UpdatedAtString() string {
|
|
||||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s DistributorState) MarshalJSON() ([]byte, error) {
|
|
||||||
type sourceJSON struct {
|
|
||||||
Manifest bundle.Manifest `json:"manifest"`
|
|
||||||
}
|
|
||||||
type stateJSON struct {
|
|
||||||
SchemaVersion int `json:"schema_version"`
|
|
||||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
|
||||||
PipelineID string `json:"pipeline_id"`
|
|
||||||
DestinationID string `json:"destination_id"`
|
|
||||||
PublishedAt string `json:"published_at"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
State StatePolicy `json:"state"`
|
|
||||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
|
||||||
Source sourceJSON `json:"source"`
|
|
||||||
Links *LinkState `json:"links,omitempty"`
|
|
||||||
Outputs []OutputFile `json:"outputs"`
|
|
||||||
}
|
|
||||||
return json.Marshal(stateJSON{
|
|
||||||
SchemaVersion: s.SchemaVersion,
|
|
||||||
DistributorVersion: s.DistributorVersion,
|
|
||||||
PipelineID: s.PipelineID,
|
|
||||||
DestinationID: s.DestinationID,
|
|
||||||
PublishedAt: s.PublishedAtString(),
|
|
||||||
CreatedAt: s.CreatedAtString(),
|
|
||||||
UpdatedAt: s.UpdatedAtString(),
|
|
||||||
State: s.State,
|
|
||||||
Reconciliation: s.Reconciliation,
|
|
||||||
Source: sourceJSON{Manifest: s.Source.Manifest},
|
|
||||||
Links: s.Links,
|
|
||||||
Outputs: s.Outputs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
|
||||||
type policyJSON struct {
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
}
|
|
||||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p ReconciliationPolicy) MarshalJSON() ([]byte, error) {
|
|
||||||
type policyJSON struct {
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
}
|
|
||||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l LinkState) MarshalJSON() ([]byte, error) {
|
|
||||||
type linkJSON struct {
|
|
||||||
PrimaryURL string `json:"primary_url,omitempty"`
|
|
||||||
}
|
|
||||||
return json.Marshal(linkJSON{PrimaryURL: l.PrimaryURL})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o OutputFile) MarshalJSON() ([]byte, error) {
|
|
||||||
type outputJSON struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
SourcePath string `json:"source_path"`
|
|
||||||
Transform string `json:"transform,omitempty"`
|
|
||||||
URL string `json:"url,omitempty"`
|
|
||||||
SHA256 string `json:"sha256"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
}
|
|
||||||
return json.Marshal(outputJSON{
|
|
||||||
Path: o.Path,
|
|
||||||
Kind: o.Kind,
|
|
||||||
SourcePath: o.SourcePath,
|
|
||||||
Transform: o.Transform,
|
|
||||||
URL: o.URL,
|
|
||||||
SHA256: o.SHA256,
|
|
||||||
Size: o.Size,
|
|
||||||
CreatedAt: o.CreatedAt.UTC().Format(time.RFC3339),
|
|
||||||
UpdatedAt: o.UpdatedAt.UTC().Format(time.RFC3339),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,473 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseValidState(t *testing.T) {
|
|
||||||
state, err := Parse([]byte(validStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
if state.SchemaVersion != SchemaVersion {
|
|
||||||
t.Fatalf("schema version = %d, want %d", state.SchemaVersion, SchemaVersion)
|
|
||||||
}
|
|
||||||
if state.PipelineID != "reports" || state.DestinationID != "archive" {
|
|
||||||
t.Fatalf("identity = %q/%q", state.PipelineID, state.DestinationID)
|
|
||||||
}
|
|
||||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("CreatedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("UpdatedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
|
||||||
t.Fatalf("state mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
|
||||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := len(state.Outputs), 1; got != want {
|
|
||||||
t.Fatalf("output count = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.Outputs[0].CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("output CreatedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseValidStateWithLinks(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"outputs": [`, `"links": {"primary_url": "https://reports.example.com/archive/report.md"},`+"\n "+`"outputs": [`, 1)
|
|
||||||
body = strings.Replace(body, `"source_path": "report.md",`, `"source_path": "report.md",`+"\n "+`"url": "https://reports.example.com/archive/report.md",`, 1)
|
|
||||||
|
|
||||||
state, err := Parse([]byte(body))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
if state.Links == nil || state.Links.PrimaryURL != "https://reports.example.com/archive/report.md" {
|
|
||||||
t.Fatalf("links = %#v, want primary URL", state.Links)
|
|
||||||
}
|
|
||||||
if state.Outputs[0].URL != "https://reports.example.com/archive/report.md" {
|
|
||||||
t.Fatalf("output URL = %q", state.Outputs[0].URL)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoveMissingOutputs(t *testing.T) {
|
|
||||||
state, err := Parse([]byte(validStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
state.Outputs = append(state.Outputs, OutputFile{
|
|
||||||
Path: "summary.txt",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "summary.txt",
|
|
||||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
||||||
Size: 10,
|
|
||||||
CreatedAt: state.CreatedAt,
|
|
||||||
UpdatedAt: state.UpdatedAt,
|
|
||||||
})
|
|
||||||
|
|
||||||
next, changed := RemoveMissingOutputs(state, []string{"summary.txt"})
|
|
||||||
if !changed {
|
|
||||||
t.Fatal("RemoveMissingOutputs() changed = false, want true")
|
|
||||||
}
|
|
||||||
if got, want := ManagedOutputPaths(next), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
unchanged, changed := RemoveMissingOutputs(next, []string{"missing.txt"})
|
|
||||||
if changed {
|
|
||||||
t.Fatal("RemoveMissingOutputs() changed = true, want false")
|
|
||||||
}
|
|
||||||
if got, want := ManagedOutputPaths(unchanged), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("unchanged paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
|
|
||||||
state, err := Parse([]byte(body))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsMissingFields(t *testing.T) {
|
|
||||||
tests := map[string]string{
|
|
||||||
"schema_version": `"schema_version"`,
|
|
||||||
"pipeline_id": `"pipeline_id"`,
|
|
||||||
"destination_id": `"destination_id"`,
|
|
||||||
"published_at": `"published_at"`,
|
|
||||||
"created_at": `"created_at"`,
|
|
||||||
"updated_at": `"updated_at"`,
|
|
||||||
"state": `"state"`,
|
|
||||||
"reconciliation": `"reconciliation"`,
|
|
||||||
"source": `"source"`,
|
|
||||||
"outputs": `"outputs"`,
|
|
||||||
}
|
|
||||||
for name, field := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), field, `"missing_`+name+`"`, 1)
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "required")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsInvalidSchemaVersion(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"schema_version": 2`, `"schema_version": 3`, 1)
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "schema_version must be 1 or 2")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseLegacyStateInfersSingleOwnerDefaults(t *testing.T) {
|
|
||||||
state, err := Parse([]byte(legacyStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := state.SchemaVersion, SchemaVersion; got != want {
|
|
||||||
t.Fatalf("schema version = %d, want normalized %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("created_at = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
|
||||||
t.Fatalf("state mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
|
||||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.Outputs[0].UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("output updated_at = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsInvalidEmbeddedManifest(t *testing.T) {
|
|
||||||
body := validStateWithManifestJSON(t, strings.Replace(manifestJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1))
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "source.manifest")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateRejectsInvalidEmbeddedManifest(t *testing.T) {
|
|
||||||
tests := map[string]func(bundle.Manifest) bundle.Manifest{
|
|
||||||
"schema version": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.SchemaVersion = 2
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"empty id": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.ID = ""
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"bad digest": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"zero created": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Created = time.Time{}
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"empty files": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Files = nil
|
|
||||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"unsafe path": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Files[0].Path = "../report.md"
|
|
||||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"duplicate path": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Files[1].Path = manifest.Files[0].Path
|
|
||||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"negative size": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Files[0].Size = -1
|
|
||||||
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
"digest mismatch": func(manifest bundle.Manifest) bundle.Manifest {
|
|
||||||
manifest.Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
|
||||||
return manifest
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, mutate := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
source := mutate(validManifest(t))
|
|
||||||
state := *withState(t, validManifest(t), func(*DistributorState) {})
|
|
||||||
state.Source.Manifest = source
|
|
||||||
err := Validate(state)
|
|
||||||
assertStateErrorContains(t, err, "source.manifest")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
tests := map[string]func(*DistributorState){
|
|
||||||
"unsafe path": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Path = "../report.md"
|
|
||||||
},
|
|
||||||
"invalid kind": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Kind = "other"
|
|
||||||
},
|
|
||||||
"invalid source": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].SourcePath = "../report.md"
|
|
||||||
},
|
|
||||||
"generated missing": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Kind = OutputKindGenerated
|
|
||||||
},
|
|
||||||
"invalid digest": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
|
|
||||||
},
|
|
||||||
"negative size": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].Size = -1
|
|
||||||
},
|
|
||||||
"missing output created_at": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].CreatedAt = time.Time{}
|
|
||||||
},
|
|
||||||
"missing output updated_at": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].UpdatedAt = time.Time{}
|
|
||||||
},
|
|
||||||
"invalid output url": func(s *DistributorState) {
|
|
||||||
s.Outputs[0].URL = "file:///tmp/report.md"
|
|
||||||
},
|
|
||||||
"invalid primary url": func(s *DistributorState) {
|
|
||||||
s.Links = &LinkState{PrimaryURL: "file:///tmp/report.md"}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, mutate := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
state := *withState(t, source, mutate)
|
|
||||||
err := Validate(state)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("Validate() error = nil, want error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateReportsURLFieldContext(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
state := *withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Links = &LinkState{PrimaryURL: "https://reports.example.com/archive#top"}
|
|
||||||
})
|
|
||||||
err := Validate(state)
|
|
||||||
assertStateErrorContains(t, err, "state links.primary_url")
|
|
||||||
assertStateErrorContains(t, err, "must not include a fragment")
|
|
||||||
|
|
||||||
state = *withState(t, source, func(s *DistributorState) {
|
|
||||||
s.Outputs[0].URL = "https://reports.example.com/archive?preview=1"
|
|
||||||
})
|
|
||||||
err = Validate(state)
|
|
||||||
assertStateErrorContains(t, err, "state outputs[0].url")
|
|
||||||
assertStateErrorContains(t, err, "must not include a query string")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "May 30"`, 1)
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "published_at must be RFC3339")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRejectsMalformedCreatedTimestamp(t *testing.T) {
|
|
||||||
body := strings.Replace(validStateJSON(t), `"created_at": "2026-05-30T11:12:00Z"`, `"created_at": "May 30"`, 1)
|
|
||||||
_, err := Parse([]byte(body))
|
|
||||||
assertStateErrorContains(t, err, "created_at must be RFC3339")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
publishedAt := time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
|
||||||
state := DistributorState{
|
|
||||||
SchemaVersion: SchemaVersion,
|
|
||||||
PipelineID: "reports",
|
|
||||||
DestinationID: "archive",
|
|
||||||
PublishedAt: publishedAt,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: SourceState{Manifest: source},
|
|
||||||
Outputs: []OutputFile{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
SHA256: source.Files[0].SHA256,
|
|
||||||
Size: source.Files[0].Size,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
data, err := json.Marshal(state)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), `"published_at":"2026-05-30T11:12:00Z"`) {
|
|
||||||
t.Fatalf("json = %s, want UTC RFC3339 published_at", data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
|
|
||||||
source := validManifest(t)
|
|
||||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
state := DistributorState{
|
|
||||||
SchemaVersion: SchemaVersion,
|
|
||||||
PipelineID: "reports",
|
|
||||||
DestinationID: "archive",
|
|
||||||
PublishedAt: publishedAt,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: SourceState{Manifest: source},
|
|
||||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
|
||||||
Outputs: []OutputFile{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
URL: "https://reports.example.com/archive/report.md",
|
|
||||||
SHA256: source.Files[0].SHA256,
|
|
||||||
Size: source.Files[0].Size,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
data, err := json.Marshal(state)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), `"links":{"primary_url":"https://reports.example.com/archive/report.md"}`) {
|
|
||||||
t.Fatalf("json = %s, want links primary URL", data)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), `"url":"https://reports.example.com/archive/report.md"`) {
|
|
||||||
t.Fatalf("json = %s, want output URL", data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOutputHelpers(t *testing.T) {
|
|
||||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
updatedAt := createdAt.Add(time.Hour)
|
|
||||||
retained := []OutputFile{{
|
|
||||||
Path: "old.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
}, {
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
}}
|
|
||||||
projected := ProjectOutputs([]OutputProjection{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
}, {
|
|
||||||
Path: "new.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
}}, retained, updatedAt)
|
|
||||||
if got, ok := FindOutputByPath(projected, "report.md"); !ok || !got.CreatedAt.Equal(createdAt) || !got.UpdatedAt.Equal(updatedAt) {
|
|
||||||
t.Fatalf("projected report.md = %#v, want preserved created_at and updated updated_at", got)
|
|
||||||
}
|
|
||||||
merged, err := MergeOutputFiles(retained, projected)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("MergeOutputFiles() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := len(merged), 3; got != want {
|
|
||||||
t.Fatalf("merged count = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := ManagedOutputPaths(DistributorState{Outputs: merged}), []string{"old.md", "report.md", "new.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("managed paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validStateJSON(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
return validStateWithManifestJSON(t, manifestJSON(t))
|
|
||||||
}
|
|
||||||
|
|
||||||
func validStateWithManifestJSON(t *testing.T, manifest string) string {
|
|
||||||
t.Helper()
|
|
||||||
return `{
|
|
||||||
"schema_version": 2,
|
|
||||||
"distributor_version": "dev",
|
|
||||||
"pipeline_id": "reports",
|
|
||||||
"destination_id": "archive",
|
|
||||||
"published_at": "2026-05-30T11:12:00Z",
|
|
||||||
"created_at": "2026-05-30T11:12:00Z",
|
|
||||||
"updated_at": "2026-05-30T11:12:00Z",
|
|
||||||
"state": {"mode": "single_owner"},
|
|
||||||
"reconciliation": {"mode": "replace"},
|
|
||||||
"source": {
|
|
||||||
"manifest": ` + manifest + `
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"path": "report.md",
|
|
||||||
"kind": "source",
|
|
||||||
"source_path": "report.md",
|
|
||||||
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
|
||||||
"size": 16,
|
|
||||||
"created_at": "2026-05-30T11:12:00Z",
|
|
||||||
"updated_at": "2026-05-30T11:12:00Z"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}`
|
|
||||||
}
|
|
||||||
|
|
||||||
func legacyStateJSON(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
return strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(validStateJSON(t),
|
|
||||||
`"schema_version": 2`, `"schema_version": 1`, 1),
|
|
||||||
` "created_at": "2026-05-30T11:12:00Z",
|
|
||||||
`, "", 1),
|
|
||||||
` "updated_at": "2026-05-30T11:12:00Z",
|
|
||||||
`, "", 1),
|
|
||||||
` "state": {"mode": "single_owner"},
|
|
||||||
`, "", 1),
|
|
||||||
` "reconciliation": {"mode": "replace"},
|
|
||||||
`, "", 1),
|
|
||||||
`,
|
|
||||||
"created_at": "2026-05-30T11:12:00Z",
|
|
||||||
"updated_at": "2026-05-30T11:12:00Z"`, "", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func manifestJSON(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile("../bundle/testdata/valid_bundle/manifest.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read manifest fixture: %v", err)
|
|
||||||
}
|
|
||||||
return string(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validManifest(t *testing.T) bundle.Manifest {
|
|
||||||
t.Helper()
|
|
||||||
manifest, err := bundle.ParseManifest([]byte(manifestJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseManifest() error = %v", err)
|
|
||||||
}
|
|
||||||
return manifest
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertStateErrorContains(t *testing.T, err error, want string) {
|
|
||||||
t.Helper()
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("error = nil, want substring %q", want)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Fatalf("error = %q, want substring %q", err.Error(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
107
internal/state/document.go
Normal file
107
internal/state/document.go
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
legacyOwnerSchema = 2
|
||||||
|
legacyMultiOwnerSchema = 3
|
||||||
|
CatalogSchemaVersion = 4
|
||||||
|
legacySchemaVersion = 1
|
||||||
|
StateModeCatalog = "catalog"
|
||||||
|
OutputKindSource = "source"
|
||||||
|
OutputKindGenerated = "generated"
|
||||||
|
)
|
||||||
|
|
||||||
|
type StateDocument struct {
|
||||||
|
Catalog *CatalogState
|
||||||
|
SupersededLegacy *SupersededLegacyState
|
||||||
|
}
|
||||||
|
|
||||||
|
type DestinationStatus struct {
|
||||||
|
Catalog *CatalogState
|
||||||
|
SupersededLegacy *SupersededLegacyState
|
||||||
|
StateErr error
|
||||||
|
HasContents bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type SupersededLegacyState struct {
|
||||||
|
SchemaVersion int
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatePolicy struct {
|
||||||
|
Mode string
|
||||||
|
}
|
||||||
|
|
||||||
|
type OwnerScope struct {
|
||||||
|
PipelineID string
|
||||||
|
DestinationID string
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawStatePolicy struct {
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseDocument(data []byte) (StateDocument, error) {
|
||||||
|
schemaVersion, err := parseSchemaVersion(data)
|
||||||
|
if err != nil {
|
||||||
|
return StateDocument{}, err
|
||||||
|
}
|
||||||
|
switch schemaVersion {
|
||||||
|
case legacySchemaVersion, legacyOwnerSchema, legacyMultiOwnerSchema:
|
||||||
|
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
|
||||||
|
case CatalogSchemaVersion:
|
||||||
|
catalog, err := ParseCatalog(data)
|
||||||
|
if err != nil {
|
||||||
|
return StateDocument{}, err
|
||||||
|
}
|
||||||
|
return StateDocument{Catalog: &catalog}, nil
|
||||||
|
default:
|
||||||
|
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSchemaVersion(data []byte) (int, error) {
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
var raw struct {
|
||||||
|
SchemaVersion *int `json:"schema_version"`
|
||||||
|
}
|
||||||
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
|
return 0, fmt.Errorf("parse distributor state: %w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := decoder.Decode(&extra); err != io.EOF {
|
||||||
|
return 0, fmt.Errorf("parse distributor state: trailing data")
|
||||||
|
}
|
||||||
|
if raw.SchemaVersion == nil {
|
||||||
|
return 0, fmt.Errorf("state schema_version is required")
|
||||||
|
}
|
||||||
|
return *raw.SchemaVersion, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
||||||
|
type policyJSON struct {
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
}
|
||||||
|
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||||
|
}
|
||||||
|
|
||||||
|
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
||||||
|
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRequiredTime(field string, raw *string) (time.Time, error) {
|
||||||
|
if raw == nil || *raw == "" {
|
||||||
|
return time.Time{}, fmt.Errorf("%s is required", field)
|
||||||
|
}
|
||||||
|
parsed, err := time.Parse(time.RFC3339, *raw)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", field, err)
|
||||||
|
}
|
||||||
|
return parsed.UTC(), nil
|
||||||
|
}
|
||||||
@@ -1,31 +1,5 @@
|
|||||||
package state
|
package state
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
)
|
|
||||||
|
|
||||||
type OutputProjection struct {
|
|
||||||
Path string
|
|
||||||
Kind string
|
|
||||||
SourcePath string
|
|
||||||
Transform string
|
|
||||||
URL string
|
|
||||||
SHA256 string
|
|
||||||
Size int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func FindOutputByPath(outputs []OutputFile, path string) (OutputFile, bool) {
|
|
||||||
for _, output := range outputs {
|
|
||||||
if output.Path == path {
|
|
||||||
return output, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return OutputFile{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
|
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
if output.Path == path {
|
if output.Path == path {
|
||||||
@@ -35,158 +9,6 @@ func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogO
|
|||||||
return CatalogOutputFile{}, false
|
return CatalogOutputFile{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func MergeOutputFiles(retained, planned []OutputFile) ([]OutputFile, error) {
|
|
||||||
outputs := make([]OutputFile, 0, len(retained)+len(planned))
|
|
||||||
indexByPath := make(map[string]int, len(retained)+len(planned))
|
|
||||||
for _, output := range retained {
|
|
||||||
if _, exists := indexByPath[output.Path]; exists {
|
|
||||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
|
||||||
}
|
|
||||||
indexByPath[output.Path] = len(outputs)
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
seenPlanned := make(map[string]struct{}, len(planned))
|
|
||||||
for _, output := range planned {
|
|
||||||
if _, exists := seenPlanned[output.Path]; exists {
|
|
||||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
|
||||||
}
|
|
||||||
seenPlanned[output.Path] = struct{}{}
|
|
||||||
if index, exists := indexByPath[output.Path]; exists {
|
|
||||||
outputs[index] = output
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
indexByPath[output.Path] = len(outputs)
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
return outputs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ManagedOutputPaths(s DistributorState) []string {
|
|
||||||
paths := make([]string, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func RemoveMissingOutputs(s DistributorState, missingPaths []string) (DistributorState, bool) {
|
|
||||||
if len(missingPaths) == 0 {
|
|
||||||
return s, false
|
|
||||||
}
|
|
||||||
missing := pathSet(missingPaths)
|
|
||||||
next := s
|
|
||||||
next.Outputs = make([]OutputFile, 0, len(s.Outputs))
|
|
||||||
changed := false
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if _, remove := missing[output.Path]; remove {
|
|
||||||
changed = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
return next, changed
|
|
||||||
}
|
|
||||||
|
|
||||||
func ProjectOutputs(outputs []OutputProjection, existing []OutputFile, now time.Time) []OutputFile {
|
|
||||||
now = now.UTC()
|
|
||||||
files := make([]OutputFile, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
createdAt := now
|
|
||||||
if existingOutput, ok := FindOutputByPath(existing, output.Path); ok {
|
|
||||||
createdAt = existingOutput.CreatedAt
|
|
||||||
}
|
|
||||||
files = append(files, OutputFile{
|
|
||||||
Path: output.Path,
|
|
||||||
Kind: output.Kind,
|
|
||||||
SourcePath: output.SourcePath,
|
|
||||||
Transform: output.Transform,
|
|
||||||
URL: output.URL,
|
|
||||||
SHA256: output.SHA256,
|
|
||||||
Size: output.Size,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: now,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
|
||||||
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) Owner(scope OwnerScope) (OwnerRecord, bool) {
|
|
||||||
for _, owner := range s.Owners {
|
|
||||||
if owner.Scope == scope {
|
|
||||||
return owner, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return OwnerRecord{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) SourceManifest(scope OwnerScope) (bundle.Manifest, bool) {
|
|
||||||
owner, ok := s.Owner(scope)
|
|
||||||
if !ok {
|
|
||||||
return bundle.Manifest{}, false
|
|
||||||
}
|
|
||||||
return owner.Source.Manifest, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) ManagedOutputPaths(scope OwnerScope) []string {
|
|
||||||
paths := make([]string, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Owner == scope {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) AllManagedOutputPaths() []string {
|
|
||||||
paths := make([]string, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func RemoveMissingSharedRootOwnerOutputs(s SharedRootState, scope OwnerScope, missingPaths []string) (SharedRootState, bool) {
|
|
||||||
if len(missingPaths) == 0 {
|
|
||||||
return s, false
|
|
||||||
}
|
|
||||||
missing := pathSet(missingPaths)
|
|
||||||
next := s
|
|
||||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
|
||||||
changed := false
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Owner == scope {
|
|
||||||
if _, remove := missing[output.Path]; remove {
|
|
||||||
changed = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
return next, changed
|
|
||||||
}
|
|
||||||
|
|
||||||
func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (SharedRootState, bool) {
|
|
||||||
if len(missingPaths) == 0 {
|
|
||||||
return s, false
|
|
||||||
}
|
|
||||||
missing := pathSet(missingPaths)
|
|
||||||
next := s
|
|
||||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
|
||||||
changed := false
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if _, remove := missing[output.Path]; remove {
|
|
||||||
changed = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
return next, changed
|
|
||||||
}
|
|
||||||
|
|
||||||
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
|
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
|
||||||
selected := make([]CatalogOutputFile, 0, len(outputs))
|
selected := make([]CatalogOutputFile, 0, len(outputs))
|
||||||
for _, output := range outputs {
|
for _, output := range outputs {
|
||||||
@@ -243,159 +65,6 @@ func RemoveMissingCatalogOutputs(s CatalogState, missingPaths []string) (Catalog
|
|||||||
return next, changed
|
return next, changed
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s SharedRootState) OutputOwner(path string) (OwnerScope, bool) {
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Path == path {
|
|
||||||
return output.Owner, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return OwnerScope{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) PathOwnershipConflict(scope OwnerScope, paths []string) (PathOwnershipConflict, bool) {
|
|
||||||
for _, path := range paths {
|
|
||||||
owner, exists := s.OutputOwner(path)
|
|
||||||
if exists && owner != scope {
|
|
||||||
return PathOwnershipConflict{
|
|
||||||
Path: path,
|
|
||||||
Owner: owner,
|
|
||||||
CurrentOwner: scope,
|
|
||||||
Detail: ComparisonDetail{
|
|
||||||
Kind: ComparisonDetailSharedRootOutputOwner,
|
|
||||||
Path: path,
|
|
||||||
CurrentOwner: scope,
|
|
||||||
ConflictingOwner: owner,
|
|
||||||
},
|
|
||||||
}, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return PathOwnershipConflict{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func ProjectSharedRootOutputs(outputs []OutputProjection, existing []SharedRootOutputFile, scope OwnerScope, source bundle.Manifest, now time.Time) []SharedRootOutputFile {
|
|
||||||
now = now.UTC()
|
|
||||||
files := make([]SharedRootOutputFile, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
createdAt := now
|
|
||||||
if existingOutput, ok := findSharedRootOutput(existing, output.Path); ok && existingOutput.Owner == scope {
|
|
||||||
createdAt = existingOutput.CreatedAt
|
|
||||||
}
|
|
||||||
files = append(files, SharedRootOutputFile{
|
|
||||||
Path: output.Path,
|
|
||||||
Kind: output.Kind,
|
|
||||||
SourcePath: output.SourcePath,
|
|
||||||
Transform: output.Transform,
|
|
||||||
URL: output.URL,
|
|
||||||
SHA256: output.SHA256,
|
|
||||||
Size: output.Size,
|
|
||||||
Owner: scope,
|
|
||||||
SourceID: source.ID,
|
|
||||||
SourceDigest: source.Digest,
|
|
||||||
SourceCreated: source.Created,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: now,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return files
|
|
||||||
}
|
|
||||||
|
|
||||||
func ReplaceOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
|
||||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
|
||||||
}
|
|
||||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
next := s
|
|
||||||
next.Owners = upsertOwner(s.Owners, owner)
|
|
||||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Owner != scope {
|
|
||||||
next.Outputs = append(next.Outputs, output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
next.Outputs = append(next.Outputs, planned...)
|
|
||||||
return next, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func MergeOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
|
||||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
|
||||||
}
|
|
||||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
next := s
|
|
||||||
next.Owners = upsertOwner(s.Owners, owner)
|
|
||||||
outputs := make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
|
||||||
indexByPath := make(map[string]int, len(s.Outputs)+len(planned))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
indexByPath[output.Path] = len(outputs)
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
for _, output := range planned {
|
|
||||||
if index, exists := indexByPath[output.Path]; exists {
|
|
||||||
outputs[index] = output
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
indexByPath[output.Path] = len(outputs)
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
next.Outputs = outputs
|
|
||||||
return next, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func findSharedRootOutput(outputs []SharedRootOutputFile, path string) (SharedRootOutputFile, bool) {
|
|
||||||
for _, output := range outputs {
|
|
||||||
if output.Path == path {
|
|
||||||
return output, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return SharedRootOutputFile{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func sharedRootOutputPaths(outputs []SharedRootOutputFile) []string {
|
|
||||||
paths := make([]string, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
paths = append(paths, output.Path)
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func rejectDuplicateSharedRootOutputs(outputs []SharedRootOutputFile) error {
|
|
||||||
seen := make(map[string]struct{}, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
if _, exists := seen[output.Path]; exists {
|
|
||||||
return fmt.Errorf("state output path %q is duplicated", output.Path)
|
|
||||||
}
|
|
||||||
seen[output.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validatePlannedSharedRootOutputs(scope OwnerScope, outputs []SharedRootOutputFile) error {
|
|
||||||
if err := rejectDuplicateSharedRootOutputs(outputs); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, output := range outputs {
|
|
||||||
if output.Owner != scope {
|
|
||||||
return fmt.Errorf("state output path %q is owned by %s/%s, not %s/%s", output.Path, output.Owner.PipelineID, output.Owner.DestinationID, scope.PipelineID, scope.DestinationID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func upsertOwner(owners []OwnerRecord, owner OwnerRecord) []OwnerRecord {
|
|
||||||
next := append([]OwnerRecord(nil), owners...)
|
|
||||||
for index, existing := range next {
|
|
||||||
if existing.Scope == owner.Scope {
|
|
||||||
next[index] = owner
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return append(next, owner)
|
|
||||||
}
|
|
||||||
|
|
||||||
func pathSet(paths []string) map[string]struct{} {
|
func pathSet(paths []string) map[string]struct{} {
|
||||||
set := make(map[string]struct{}, len(paths))
|
set := make(map[string]struct{}, len(paths))
|
||||||
for _, path := range paths {
|
for _, path := range paths {
|
||||||
|
|||||||
@@ -22,33 +22,6 @@ type PrunePlan struct {
|
|||||||
Preserved []PruneCandidate
|
Preserved []PruneCandidate
|
||||||
}
|
}
|
||||||
|
|
||||||
func SingleOwnerPruneCandidates(s DistributorState) []PruneCandidate {
|
|
||||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
candidates = append(candidates, PruneCandidate{
|
|
||||||
Path: output.Path,
|
|
||||||
UpdatedAt: output.UpdatedAt,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return candidates
|
|
||||||
}
|
|
||||||
|
|
||||||
func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandidate {
|
|
||||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
|
||||||
for _, output := range s.Outputs {
|
|
||||||
if output.Owner != scope {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
owner := output.Owner
|
|
||||||
candidates = append(candidates, PruneCandidate{
|
|
||||||
Path: output.Path,
|
|
||||||
UpdatedAt: output.UpdatedAt,
|
|
||||||
Owner: &owner,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return candidates
|
|
||||||
}
|
|
||||||
|
|
||||||
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
|
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
|
||||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||||
for _, output := range s.Outputs {
|
for _, output := range s.Outputs {
|
||||||
|
|||||||
@@ -74,19 +74,6 @@ func TestPlanPruneDeterministicTieBreaking(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSharedRootPruneCandidatesPreserveOtherOwners(t *testing.T) {
|
|
||||||
sharedRoot := validSharedRootState(t)
|
|
||||||
scope := CurrentOwnerScope("reports", "archive")
|
|
||||||
candidates := SharedRootPruneCandidates(sharedRoot, scope)
|
|
||||||
|
|
||||||
if got, want := pruneCandidatePaths(candidates), "report.md"; got != want {
|
|
||||||
t.Fatalf("candidates = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if candidates[0].Owner == nil || *candidates[0].Owner != scope {
|
|
||||||
t.Fatalf("candidate owner = %#v, want current owner", candidates[0].Owner)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func pruneCandidatePaths(candidates []PruneCandidate) string {
|
func pruneCandidatePaths(candidates []PruneCandidate) string {
|
||||||
paths := make([]string, 0, len(candidates))
|
paths := make([]string, 0, len(candidates))
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
|
|||||||
@@ -1,531 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
type StateDocument struct {
|
|
||||||
SingleOwner *DistributorState
|
|
||||||
SharedRoot *SharedRootState
|
|
||||||
Catalog *CatalogState
|
|
||||||
SupersededLegacy *SupersededLegacyState
|
|
||||||
}
|
|
||||||
|
|
||||||
type SupersededLegacyState struct {
|
|
||||||
SchemaVersion int
|
|
||||||
}
|
|
||||||
|
|
||||||
type SharedRootState struct {
|
|
||||||
SchemaVersion int
|
|
||||||
DistributorVersion string
|
|
||||||
CreatedAt time.Time
|
|
||||||
UpdatedAt time.Time
|
|
||||||
State StatePolicy
|
|
||||||
Owners []OwnerRecord
|
|
||||||
Outputs []SharedRootOutputFile
|
|
||||||
}
|
|
||||||
|
|
||||||
type OwnerScope struct {
|
|
||||||
PipelineID string
|
|
||||||
DestinationID string
|
|
||||||
}
|
|
||||||
|
|
||||||
type OwnerRecord struct {
|
|
||||||
Scope OwnerScope
|
|
||||||
Reconciliation ReconciliationPolicy
|
|
||||||
Source SourceState
|
|
||||||
Links *LinkState
|
|
||||||
}
|
|
||||||
|
|
||||||
type SharedRootOutputFile struct {
|
|
||||||
Path string
|
|
||||||
Kind string
|
|
||||||
SourcePath string
|
|
||||||
Transform string
|
|
||||||
URL string
|
|
||||||
SHA256 string
|
|
||||||
Size int64
|
|
||||||
Owner OwnerScope
|
|
||||||
SourceID string
|
|
||||||
SourceDigest string
|
|
||||||
SourceCreated time.Time
|
|
||||||
CreatedAt time.Time
|
|
||||||
UpdatedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type PathOwnershipConflict struct {
|
|
||||||
Path string
|
|
||||||
Owner OwnerScope
|
|
||||||
CurrentOwner OwnerScope
|
|
||||||
Detail ComparisonDetail
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawSharedRootState struct {
|
|
||||||
SchemaVersion *int `json:"schema_version"`
|
|
||||||
DistributorVersion string `json:"distributor_version"`
|
|
||||||
CreatedAt *string `json:"created_at"`
|
|
||||||
UpdatedAt *string `json:"updated_at"`
|
|
||||||
State *rawStatePolicy `json:"state"`
|
|
||||||
Owners []rawOwnerRecord `json:"owners"`
|
|
||||||
Outputs []rawSharedRootOutput `json:"outputs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawOwnerRecord struct {
|
|
||||||
PipelineID *string `json:"pipeline_id"`
|
|
||||||
DestinationID *string `json:"destination_id"`
|
|
||||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
|
||||||
Source *rawSourceState `json:"source"`
|
|
||||||
Links *rawLinkState `json:"links"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type rawSharedRootOutput struct {
|
|
||||||
Path *string `json:"path"`
|
|
||||||
Kind *string `json:"kind"`
|
|
||||||
SourcePath *string `json:"source_path"`
|
|
||||||
Transform string `json:"transform"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
SHA256 *string `json:"sha256"`
|
|
||||||
Size *int64 `json:"size"`
|
|
||||||
PipelineID *string `json:"pipeline_id"`
|
|
||||||
DestinationID *string `json:"destination_id"`
|
|
||||||
SourceID *string `json:"source_id"`
|
|
||||||
SourceDigest *string `json:"source_digest"`
|
|
||||||
SourceCreated *string `json:"source_created"`
|
|
||||||
CreatedAt *string `json:"created_at"`
|
|
||||||
UpdatedAt *string `json:"updated_at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseDocument(data []byte) (StateDocument, error) {
|
|
||||||
schemaVersion, err := parseSchemaVersion(data)
|
|
||||||
if err != nil {
|
|
||||||
return StateDocument{}, err
|
|
||||||
}
|
|
||||||
switch schemaVersion {
|
|
||||||
case legacySchemaVersion, SchemaVersion, SharedRootSchemaVersion:
|
|
||||||
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
|
|
||||||
case CatalogSchemaVersion:
|
|
||||||
catalog, err := ParseCatalog(data)
|
|
||||||
if err != nil {
|
|
||||||
return StateDocument{}, err
|
|
||||||
}
|
|
||||||
return StateDocument{Catalog: &catalog}, nil
|
|
||||||
default:
|
|
||||||
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSchemaVersion(data []byte) (int, error) {
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
||||||
var raw struct {
|
|
||||||
SchemaVersion *int `json:"schema_version"`
|
|
||||||
}
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return 0, fmt.Errorf("parse distributor state: %w", err)
|
|
||||||
}
|
|
||||||
var extra any
|
|
||||||
if err := decoder.Decode(&extra); err != io.EOF {
|
|
||||||
return 0, fmt.Errorf("parse distributor state: trailing data")
|
|
||||||
}
|
|
||||||
if raw.SchemaVersion == nil {
|
|
||||||
return 0, fmt.Errorf("state schema_version is required")
|
|
||||||
}
|
|
||||||
return *raw.SchemaVersion, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseSharedRoot(data []byte) (SharedRootState, error) {
|
|
||||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
||||||
var raw rawSharedRootState
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return SharedRootState{}, fmt.Errorf("parse distributor state: %w", err)
|
|
||||||
}
|
|
||||||
var extra any
|
|
||||||
if err := decoder.Decode(&extra); err != io.EOF {
|
|
||||||
return SharedRootState{}, fmt.Errorf("parse distributor state: trailing data")
|
|
||||||
}
|
|
||||||
state, err := parseSharedRootRaw(raw)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
if err := ValidateSharedRoot(state); err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSharedRootRaw(raw rawSharedRootState) (SharedRootState, error) {
|
|
||||||
if raw.SchemaVersion == nil {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state schema_version is required")
|
|
||||||
}
|
|
||||||
state := SharedRootState{SchemaVersion: *raw.SchemaVersion}
|
|
||||||
if state.SchemaVersion != SharedRootSchemaVersion {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
|
||||||
}
|
|
||||||
state.DistributorVersion = raw.DistributorVersion
|
|
||||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
state.CreatedAt = createdAt
|
|
||||||
state.UpdatedAt = updatedAt
|
|
||||||
if raw.State == nil || raw.State.Mode == "" {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state state.mode is required")
|
|
||||||
}
|
|
||||||
state.State.Mode = raw.State.Mode
|
|
||||||
if raw.Owners == nil {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state owners is required")
|
|
||||||
}
|
|
||||||
owners, err := parseOwnerRecords(raw.Owners)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
state.Owners = owners
|
|
||||||
if raw.Outputs == nil {
|
|
||||||
return SharedRootState{}, fmt.Errorf("state outputs is required")
|
|
||||||
}
|
|
||||||
outputs, err := parseSharedRootOutputs(raw.Outputs)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootState{}, err
|
|
||||||
}
|
|
||||||
state.Outputs = outputs
|
|
||||||
return state, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOwnerRecords(rawOwners []rawOwnerRecord) ([]OwnerRecord, error) {
|
|
||||||
owners := make([]OwnerRecord, 0, len(rawOwners))
|
|
||||||
for index, raw := range rawOwners {
|
|
||||||
owner, err := parseOwnerRecord(index, raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
owners = append(owners, owner)
|
|
||||||
}
|
|
||||||
return owners, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseOwnerRecord(index int, raw rawOwnerRecord) (OwnerRecord, error) {
|
|
||||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].destination_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].reconciliation.mode is required", index)
|
|
||||||
}
|
|
||||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest is required", index)
|
|
||||||
}
|
|
||||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
|
||||||
if err != nil {
|
|
||||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
|
||||||
}
|
|
||||||
owner := OwnerRecord{
|
|
||||||
Scope: OwnerScope{
|
|
||||||
PipelineID: *raw.PipelineID,
|
|
||||||
DestinationID: *raw.DestinationID,
|
|
||||||
},
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: raw.Reconciliation.Mode},
|
|
||||||
Source: SourceState{Manifest: manifest},
|
|
||||||
}
|
|
||||||
if raw.Links != nil {
|
|
||||||
owner.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
|
||||||
}
|
|
||||||
return owner, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSharedRootOutputs(rawOutputs []rawSharedRootOutput) ([]SharedRootOutputFile, error) {
|
|
||||||
outputs := make([]SharedRootOutputFile, 0, len(rawOutputs))
|
|
||||||
for index, raw := range rawOutputs {
|
|
||||||
output, err := parseSharedRootOutput(index, raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
return outputs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSharedRootOutput(index int, raw rawSharedRootOutput) (SharedRootOutputFile, error) {
|
|
||||||
if raw.Path == nil || *raw.Path == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
|
||||||
}
|
|
||||||
if raw.Kind == nil || *raw.Kind == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
|
||||||
}
|
|
||||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
|
||||||
}
|
|
||||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
|
||||||
}
|
|
||||||
if raw.Size == nil {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
|
||||||
}
|
|
||||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.SourceID == nil || *raw.SourceID == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_id is required", index)
|
|
||||||
}
|
|
||||||
if raw.SourceDigest == nil || *raw.SourceDigest == "" {
|
|
||||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_digest is required", index)
|
|
||||||
}
|
|
||||||
sourceCreated, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source_created", index), raw.SourceCreated)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootOutputFile{}, err
|
|
||||||
}
|
|
||||||
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootOutputFile{}, err
|
|
||||||
}
|
|
||||||
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
|
||||||
if err != nil {
|
|
||||||
return SharedRootOutputFile{}, err
|
|
||||||
}
|
|
||||||
return SharedRootOutputFile{
|
|
||||||
Path: *raw.Path,
|
|
||||||
Kind: *raw.Kind,
|
|
||||||
SourcePath: *raw.SourcePath,
|
|
||||||
Transform: raw.Transform,
|
|
||||||
URL: raw.URL,
|
|
||||||
SHA256: *raw.SHA256,
|
|
||||||
Size: *raw.Size,
|
|
||||||
Owner: OwnerScope{
|
|
||||||
PipelineID: *raw.PipelineID,
|
|
||||||
DestinationID: *raw.DestinationID,
|
|
||||||
},
|
|
||||||
SourceID: *raw.SourceID,
|
|
||||||
SourceDigest: *raw.SourceDigest,
|
|
||||||
SourceCreated: sourceCreated,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) CreatedAtString() string {
|
|
||||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) UpdatedAtString() string {
|
|
||||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o SharedRootOutputFile) SourceCreatedString() string {
|
|
||||||
return o.SourceCreated.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o SharedRootOutputFile) CreatedAtString() string {
|
|
||||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o SharedRootOutputFile) UpdatedAtString() string {
|
|
||||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s SharedRootState) MarshalJSON() ([]byte, error) {
|
|
||||||
type stateJSON struct {
|
|
||||||
SchemaVersion int `json:"schema_version"`
|
|
||||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
State StatePolicy `json:"state"`
|
|
||||||
Owners []OwnerRecord `json:"owners"`
|
|
||||||
Outputs []SharedRootOutputFile `json:"outputs"`
|
|
||||||
}
|
|
||||||
return json.Marshal(stateJSON{
|
|
||||||
SchemaVersion: s.SchemaVersion,
|
|
||||||
DistributorVersion: s.DistributorVersion,
|
|
||||||
CreatedAt: s.CreatedAtString(),
|
|
||||||
UpdatedAt: s.UpdatedAtString(),
|
|
||||||
State: s.State,
|
|
||||||
Owners: s.Owners,
|
|
||||||
Outputs: s.Outputs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o OwnerRecord) MarshalJSON() ([]byte, error) {
|
|
||||||
type sourceJSON struct {
|
|
||||||
Manifest bundle.Manifest `json:"manifest"`
|
|
||||||
}
|
|
||||||
type ownerJSON struct {
|
|
||||||
PipelineID string `json:"pipeline_id"`
|
|
||||||
DestinationID string `json:"destination_id"`
|
|
||||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
|
||||||
Source sourceJSON `json:"source"`
|
|
||||||
Links *LinkState `json:"links,omitempty"`
|
|
||||||
}
|
|
||||||
return json.Marshal(ownerJSON{
|
|
||||||
PipelineID: o.Scope.PipelineID,
|
|
||||||
DestinationID: o.Scope.DestinationID,
|
|
||||||
Reconciliation: o.Reconciliation,
|
|
||||||
Source: sourceJSON{Manifest: o.Source.Manifest},
|
|
||||||
Links: o.Links,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o SharedRootOutputFile) MarshalJSON() ([]byte, error) {
|
|
||||||
type outputJSON struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
SourcePath string `json:"source_path"`
|
|
||||||
Transform string `json:"transform,omitempty"`
|
|
||||||
URL string `json:"url,omitempty"`
|
|
||||||
SHA256 string `json:"sha256"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
PipelineID string `json:"pipeline_id"`
|
|
||||||
DestinationID string `json:"destination_id"`
|
|
||||||
SourceID string `json:"source_id"`
|
|
||||||
SourceDigest string `json:"source_digest"`
|
|
||||||
SourceCreated string `json:"source_created"`
|
|
||||||
CreatedAt string `json:"created_at"`
|
|
||||||
UpdatedAt string `json:"updated_at"`
|
|
||||||
}
|
|
||||||
return json.Marshal(outputJSON{
|
|
||||||
Path: o.Path,
|
|
||||||
Kind: o.Kind,
|
|
||||||
SourcePath: o.SourcePath,
|
|
||||||
Transform: o.Transform,
|
|
||||||
URL: o.URL,
|
|
||||||
SHA256: o.SHA256,
|
|
||||||
Size: o.Size,
|
|
||||||
PipelineID: o.Owner.PipelineID,
|
|
||||||
DestinationID: o.Owner.DestinationID,
|
|
||||||
SourceID: o.SourceID,
|
|
||||||
SourceDigest: o.SourceDigest,
|
|
||||||
SourceCreated: o.SourceCreatedString(),
|
|
||||||
CreatedAt: o.CreatedAtString(),
|
|
||||||
UpdatedAt: o.UpdatedAtString(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func ValidateSharedRoot(s SharedRootState) error {
|
|
||||||
if s.SchemaVersion != SharedRootSchemaVersion {
|
|
||||||
return fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
|
||||||
}
|
|
||||||
if s.CreatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state created_at is required")
|
|
||||||
}
|
|
||||||
if s.UpdatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state updated_at is required")
|
|
||||||
}
|
|
||||||
if s.State.Mode != StateModeSharedRoot {
|
|
||||||
return fmt.Errorf("state state.mode must be %s", StateModeSharedRoot)
|
|
||||||
}
|
|
||||||
if s.Owners == nil {
|
|
||||||
return fmt.Errorf("state owners is required")
|
|
||||||
}
|
|
||||||
owners := make(map[OwnerScope]OwnerRecord, len(s.Owners))
|
|
||||||
for index, owner := range s.Owners {
|
|
||||||
if err := validateOwnerRecord(index, owner); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, exists := owners[owner.Scope]; exists {
|
|
||||||
return fmt.Errorf("state owners[%d] duplicates owner %s/%s", index, owner.Scope.PipelineID, owner.Scope.DestinationID)
|
|
||||||
}
|
|
||||||
owners[owner.Scope] = owner
|
|
||||||
}
|
|
||||||
if s.Outputs == nil {
|
|
||||||
return fmt.Errorf("state outputs is required")
|
|
||||||
}
|
|
||||||
seenPaths := make(map[string]struct{}, len(s.Outputs))
|
|
||||||
for index, output := range s.Outputs {
|
|
||||||
if err := validateSharedRootOutput(index, output, owners); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, exists := seenPaths[output.Path]; exists {
|
|
||||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
|
||||||
}
|
|
||||||
seenPaths[output.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateOwnerRecord(index int, owner OwnerRecord) error {
|
|
||||||
if owner.Scope.PipelineID == "" {
|
|
||||||
return fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
|
||||||
}
|
|
||||||
if owner.Scope.DestinationID == "" {
|
|
||||||
return fmt.Errorf("state owners[%d].destination_id is required", index)
|
|
||||||
}
|
|
||||||
if owner.Reconciliation.Mode != config.ReconciliationModeReplace && owner.Reconciliation.Mode != config.ReconciliationModeMerge {
|
|
||||||
return fmt.Errorf("state owners[%d].reconciliation.mode must be %s or %s", index, config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
|
||||||
}
|
|
||||||
if err := validateEmbeddedManifest(owner.Source.Manifest); err != nil {
|
|
||||||
return fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
|
||||||
}
|
|
||||||
if owner.Links != nil && owner.Links.PrimaryURL != "" {
|
|
||||||
if err := link.ValidateHTTPURL(owner.Links.PrimaryURL); err != nil {
|
|
||||||
return fmt.Errorf("state owners[%d].links.primary_url: %w", index, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateSharedRootOutput(index int, output SharedRootOutputFile, owners map[OwnerScope]OwnerRecord) error {
|
|
||||||
if err := storage.ValidatePath(output.Path); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
|
||||||
}
|
|
||||||
switch output.Kind {
|
|
||||||
case OutputKindSource, OutputKindGenerated:
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
|
||||||
}
|
|
||||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
|
||||||
}
|
|
||||||
if output.URL != "" {
|
|
||||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.Size < 0 {
|
|
||||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
|
||||||
}
|
|
||||||
if output.Owner.PipelineID == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
|
||||||
}
|
|
||||||
if output.Owner.DestinationID == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].destination_id is required", index)
|
|
||||||
}
|
|
||||||
if _, exists := owners[output.Owner]; !exists {
|
|
||||||
return fmt.Errorf("state outputs[%d] references unknown owner %s/%s", index, output.Owner.PipelineID, output.Owner.DestinationID)
|
|
||||||
}
|
|
||||||
if output.SourceID == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_id is required", index)
|
|
||||||
}
|
|
||||||
if err := bundle.ValidateDigest(output.SourceDigest); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_digest: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.SourceCreated.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_created is required", index)
|
|
||||||
}
|
|
||||||
if output.CreatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
|
||||||
}
|
|
||||||
if output.UpdatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,314 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseSharedRootState(t *testing.T) {
|
|
||||||
state, err := ParseSharedRoot([]byte(validSharedRootStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseSharedRoot() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := state.SchemaVersion, SharedRootSchemaVersion; got != want {
|
|
||||||
t.Fatalf("schema version = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
|
||||||
t.Fatalf("created_at = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.State.Mode, StateModeSharedRoot; got != want {
|
|
||||||
t.Fatalf("state mode = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
if got, want := len(state.Owners), 2; got != want {
|
|
||||||
t.Fatalf("owner count = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
if got, want := len(state.Outputs), 2; got != want {
|
|
||||||
t.Fatalf("output count = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
scope := CurrentOwnerScope("reports", "archive")
|
|
||||||
manifest, ok := state.SourceManifest(scope)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("SourceManifest() ok = false, want true")
|
|
||||||
}
|
|
||||||
if got, want := manifest.ID, validManifest(t).ID; got != want {
|
|
||||||
t.Fatalf("source manifest id = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseSharedRootRejectsInvalidMetadata(t *testing.T) {
|
|
||||||
tests := map[string]func(string) string{
|
|
||||||
"schema": func(body string) string {
|
|
||||||
return strings.Replace(body, `"schema_version": 3`, `"schema_version": 2`, 1)
|
|
||||||
},
|
|
||||||
"state mode": func(body string) string {
|
|
||||||
return strings.Replace(body, `"mode": "shared_root"`, `"mode": "single_owner"`, 1)
|
|
||||||
},
|
|
||||||
"duplicate owner": func(body string) string {
|
|
||||||
return strings.Replace(body, `"destination_id": "html"`, `"destination_id": "archive"`, 1)
|
|
||||||
},
|
|
||||||
"unknown output owner": func(body string) string {
|
|
||||||
return strings.Replace(body, `"destination_id": "html",`, `"destination_id": "missing",`, 1)
|
|
||||||
},
|
|
||||||
"duplicate output": func(body string) string {
|
|
||||||
return strings.Replace(body, `"path": "report.html"`, `"path": "report.md"`, 1)
|
|
||||||
},
|
|
||||||
"invalid source digest": func(body string) string {
|
|
||||||
return strings.Replace(body, `"source_digest": "sha256:`, `"source_digest": "SHA256:`, 1)
|
|
||||||
},
|
|
||||||
"invalid owner link": func(body string) string {
|
|
||||||
return strings.Replace(body, `"primary_url": "https://reports.example.com/archive/report.md"`, `"primary_url": "file:///tmp/report.md"`, 1)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for name, mutate := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
_, err := ParseSharedRoot([]byte(mutate(validSharedRootStateJSON(t))))
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("ParseSharedRoot() error = nil, want error")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedRootMarshalNormalizesTimestamps(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
state.CreatedAt = time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
|
||||||
state.UpdatedAt = state.CreatedAt
|
|
||||||
state.Outputs[0].SourceCreated = state.CreatedAt
|
|
||||||
state.Outputs[0].CreatedAt = state.CreatedAt
|
|
||||||
state.Outputs[0].UpdatedAt = state.CreatedAt
|
|
||||||
|
|
||||||
data, err := json.Marshal(state)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Marshal() error = %v", err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{
|
|
||||||
`"created_at":"2026-05-30T11:12:00Z"`,
|
|
||||||
`"updated_at":"2026-05-30T11:12:00Z"`,
|
|
||||||
`"source_created":"2026-05-30T11:12:00Z"`,
|
|
||||||
} {
|
|
||||||
if !strings.Contains(string(data), want) {
|
|
||||||
t.Fatalf("json = %s, want %s", data, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedRootOutputHelpers(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
archive := CurrentOwnerScope("reports", "archive")
|
|
||||||
html := CurrentOwnerScope("reports", "html")
|
|
||||||
|
|
||||||
if got, want := state.ManagedOutputPaths(archive), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("archive paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
if got, want := state.AllManagedOutputPaths(), []string{"report.md", "report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("all paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
conflict, ok := state.PathOwnershipConflict(archive, []string{"report.html"})
|
|
||||||
if !ok || conflict.Owner != html {
|
|
||||||
t.Fatalf("conflict = %#v ok=%t, want html owner conflict", conflict, ok)
|
|
||||||
}
|
|
||||||
if conflict.Detail.Kind != ComparisonDetailSharedRootOutputOwner {
|
|
||||||
t.Fatalf("conflict detail kind = %q, want %q", conflict.Detail.Kind, ComparisonDetailSharedRootOutputOwner)
|
|
||||||
}
|
|
||||||
if conflict.Detail.Path != "report.html" || conflict.Detail.CurrentOwner != archive || conflict.Detail.ConflictingOwner != html {
|
|
||||||
t.Fatalf("conflict detail = %#v, want path and owners", conflict.Detail)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoveMissingSharedRootOwnerOutputs(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
archive := CurrentOwnerScope("reports", "archive")
|
|
||||||
next, changed := RemoveMissingSharedRootOwnerOutputs(state, archive, []string{"report.md", "report.html"})
|
|
||||||
if !changed {
|
|
||||||
t.Fatal("RemoveMissingSharedRootOwnerOutputs() changed = false, want true")
|
|
||||||
}
|
|
||||||
if got, want := next.AllManagedOutputPaths(), []string{"report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
if _, ok := next.OutputOwner("report.html"); !ok {
|
|
||||||
t.Fatal("report.html owner missing, want unrelated owner preserved")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoveMissingSharedRootOutputs(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
next, changed := RemoveMissingSharedRootOutputs(state, []string{"report.md", "report.html"})
|
|
||||||
if !changed {
|
|
||||||
t.Fatal("RemoveMissingSharedRootOutputs() changed = false, want true")
|
|
||||||
}
|
|
||||||
if got := next.AllManagedOutputPaths(); len(got) != 0 {
|
|
||||||
t.Fatalf("paths = %#v, want none", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedRootProjectAndMergeOwnerOutputs(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
archive := CurrentOwnerScope("reports", "archive")
|
|
||||||
owner, ok := state.Owner(archive)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("Owner() ok = false, want true")
|
|
||||||
}
|
|
||||||
now := time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC)
|
|
||||||
planned := ProjectSharedRootOutputs([]OutputProjection{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
SHA256: validManifest(t).Files[0].SHA256,
|
|
||||||
Size: validManifest(t).Files[0].Size,
|
|
||||||
}, {
|
|
||||||
Path: "summary.txt",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "summary.txt",
|
|
||||||
SHA256: validManifest(t).Files[1].SHA256,
|
|
||||||
Size: validManifest(t).Files[1].Size,
|
|
||||||
}}, state.Outputs, archive, validManifest(t), now)
|
|
||||||
|
|
||||||
merged, err := MergeOwnerOutputs(state, archive, owner, planned)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("MergeOwnerOutputs() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := merged.AllManagedOutputPaths(), []string{"report.md", "report.html", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("merged paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
if merged.Outputs[0].CreatedAt.Equal(now) {
|
|
||||||
t.Fatalf("merged updated output created_at = %s, want preserved timestamp", merged.Outputs[0].CreatedAt)
|
|
||||||
}
|
|
||||||
if !merged.Outputs[0].UpdatedAt.Equal(now) {
|
|
||||||
t.Fatalf("merged updated output updated_at = %s, want %s", merged.Outputs[0].UpdatedAt, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
replaced, err := ReplaceOwnerOutputs(state, archive, owner, planned)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReplaceOwnerOutputs() error = %v", err)
|
|
||||||
}
|
|
||||||
if got, want := replaced.AllManagedOutputPaths(), []string{"report.html", "report.md", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
|
||||||
t.Fatalf("replaced paths = %#v, want %#v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedRootOwnerOutputHelpersRejectConflicts(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
archive := CurrentOwnerScope("reports", "archive")
|
|
||||||
owner, ok := state.Owner(archive)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("Owner() ok = false, want true")
|
|
||||||
}
|
|
||||||
planned := []SharedRootOutputFile{{
|
|
||||||
Path: "report.html",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
Owner: archive,
|
|
||||||
CreatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
|
||||||
UpdatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
|
||||||
}}
|
|
||||||
|
|
||||||
if _, err := MergeOwnerOutputs(state, archive, owner, planned); err == nil {
|
|
||||||
t.Fatal("MergeOwnerOutputs() error = nil, want owner conflict")
|
|
||||||
}
|
|
||||||
if _, err := ReplaceOwnerOutputs(state, archive, owner, planned); err == nil {
|
|
||||||
t.Fatal("ReplaceOwnerOutputs() error = nil, want owner conflict")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompareSharedRootOwnerScopesCurrentOwner(t *testing.T) {
|
|
||||||
state := validSharedRootState(t)
|
|
||||||
source := validManifest(t)
|
|
||||||
source.Created = source.Created.Add(time.Hour)
|
|
||||||
scope := CurrentOwnerScope("reports", "archive")
|
|
||||||
|
|
||||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{SharedRoot: &state, HasContents: true})
|
|
||||||
if comparison.Outcome != OutcomeDestinationOlder {
|
|
||||||
t.Fatalf("comparison = %#v, want destination older for current owner", comparison)
|
|
||||||
}
|
|
||||||
|
|
||||||
missing := CompareSharedRootOwner(source, CurrentOwnerScope("missing", "archive"), DestinationStatus{SharedRoot: &state, HasContents: true})
|
|
||||||
if missing.Outcome != OutcomeDestinationAbsent {
|
|
||||||
t.Fatalf("missing owner comparison = %#v, want destination absent", missing)
|
|
||||||
}
|
|
||||||
if missing.Detail.Kind != ComparisonDetailSharedRootOwnerAbsent {
|
|
||||||
t.Fatalf("missing owner detail kind = %q, want %q", missing.Detail.Kind, ComparisonDetailSharedRootOwnerAbsent)
|
|
||||||
}
|
|
||||||
if missing.Detail.CurrentOwner != CurrentOwnerScope("missing", "archive") {
|
|
||||||
t.Fatalf("missing owner detail = %#v, want missing/archive", missing.Detail)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCompareSharedRootOwnerAcceptsMatchingSingleOwnerState(t *testing.T) {
|
|
||||||
singleOwner, err := Parse([]byte(validStateJSON(t)))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Parse() error = %v", err)
|
|
||||||
}
|
|
||||||
source := singleOwner.Source.Manifest
|
|
||||||
source.Created = source.Created.Add(time.Hour)
|
|
||||||
scope := CurrentOwnerScope(singleOwner.PipelineID, singleOwner.DestinationID)
|
|
||||||
|
|
||||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{State: &singleOwner, HasContents: true})
|
|
||||||
if comparison.Outcome != OutcomeDestinationOlder {
|
|
||||||
t.Fatalf("comparison = %#v, want destination older", comparison)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validSharedRootStateJSON(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
data, err := json.MarshalIndent(validSharedRootState(t), "", " ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal shared root state: %v", err)
|
|
||||||
}
|
|
||||||
return string(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validSharedRootState(t *testing.T) SharedRootState {
|
|
||||||
t.Helper()
|
|
||||||
source := validManifest(t)
|
|
||||||
htmlSource := source
|
|
||||||
htmlSource.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
|
||||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
return SharedRootState{
|
|
||||||
SchemaVersion: SharedRootSchemaVersion,
|
|
||||||
DistributorVersion: "dev",
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
State: StatePolicy{Mode: StateModeSharedRoot},
|
|
||||||
Owners: []OwnerRecord{{
|
|
||||||
Scope: CurrentOwnerScope("reports", "archive"),
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: SourceState{Manifest: source},
|
|
||||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
|
||||||
}, {
|
|
||||||
Scope: CurrentOwnerScope("reports", "html"),
|
|
||||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
|
||||||
Source: SourceState{Manifest: htmlSource},
|
|
||||||
}},
|
|
||||||
Outputs: []SharedRootOutputFile{{
|
|
||||||
Path: "report.md",
|
|
||||||
Kind: OutputKindSource,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
SHA256: source.Files[0].SHA256,
|
|
||||||
Size: source.Files[0].Size,
|
|
||||||
Owner: CurrentOwnerScope("reports", "archive"),
|
|
||||||
SourceID: source.ID,
|
|
||||||
SourceDigest: source.Digest,
|
|
||||||
SourceCreated: source.Created,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
}, {
|
|
||||||
Path: "report.html",
|
|
||||||
Kind: OutputKindGenerated,
|
|
||||||
SourcePath: "report.md",
|
|
||||||
Transform: "markdown_to_html",
|
|
||||||
URL: "https://reports.example.com/html/report.html",
|
|
||||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
||||||
Size: 128,
|
|
||||||
Owner: CurrentOwnerScope("reports", "html"),
|
|
||||||
SourceID: htmlSource.ID,
|
|
||||||
SourceDigest: htmlSource.Digest,
|
|
||||||
SourceCreated: htmlSource.Created,
|
|
||||||
CreatedAt: createdAt,
|
|
||||||
UpdatedAt: createdAt,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,103 +0,0 @@
|
|||||||
package state
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
OutputKindSource = "source"
|
|
||||||
OutputKindGenerated = "generated"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Validate(s DistributorState) error {
|
|
||||||
if s.SchemaVersion != SchemaVersion {
|
|
||||||
return fmt.Errorf("state schema_version must be %d", SchemaVersion)
|
|
||||||
}
|
|
||||||
if s.PipelineID == "" {
|
|
||||||
return fmt.Errorf("state pipeline_id is required")
|
|
||||||
}
|
|
||||||
if s.DestinationID == "" {
|
|
||||||
return fmt.Errorf("state destination_id is required")
|
|
||||||
}
|
|
||||||
if s.PublishedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state published_at is required")
|
|
||||||
}
|
|
||||||
if s.CreatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state created_at is required")
|
|
||||||
}
|
|
||||||
if s.UpdatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state updated_at is required")
|
|
||||||
}
|
|
||||||
if s.State.Mode != StateModeSingleOwner {
|
|
||||||
return fmt.Errorf("state state.mode must be %s", StateModeSingleOwner)
|
|
||||||
}
|
|
||||||
if s.Reconciliation.Mode != config.ReconciliationModeReplace && s.Reconciliation.Mode != config.ReconciliationModeMerge {
|
|
||||||
return fmt.Errorf("state reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
|
||||||
}
|
|
||||||
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
|
|
||||||
return fmt.Errorf("state source.manifest: %w", err)
|
|
||||||
}
|
|
||||||
if s.Links != nil && s.Links.PrimaryURL != "" {
|
|
||||||
if err := link.ValidateHTTPURL(s.Links.PrimaryURL); err != nil {
|
|
||||||
return fmt.Errorf("state links.primary_url: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if s.Outputs == nil {
|
|
||||||
return fmt.Errorf("state outputs is required")
|
|
||||||
}
|
|
||||||
seen := make(map[string]struct{}, len(s.Outputs))
|
|
||||||
for index, output := range s.Outputs {
|
|
||||||
if err := validateOutput(index, output); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, exists := seen[output.Path]; exists {
|
|
||||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
|
||||||
}
|
|
||||||
seen[output.Path] = struct{}{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateEmbeddedManifest(manifest bundle.Manifest) error {
|
|
||||||
return bundle.ValidateManifest(manifest)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateOutput(index int, output OutputFile) error {
|
|
||||||
if err := storage.ValidatePath(output.Path); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
|
||||||
}
|
|
||||||
switch output.Kind {
|
|
||||||
case OutputKindSource, OutputKindGenerated:
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
|
||||||
}
|
|
||||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
|
||||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
|
||||||
}
|
|
||||||
if output.URL != "" {
|
|
||||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
|
||||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
|
||||||
}
|
|
||||||
if output.Size < 0 {
|
|
||||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
|
||||||
}
|
|
||||||
if output.CreatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
|
||||||
}
|
|
||||||
if output.UpdatedAt.IsZero() {
|
|
||||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -11,8 +11,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
)
|
)
|
||||||
@@ -33,13 +31,6 @@ type BundleOptions struct {
|
|||||||
ExtraFiles []SourceFile
|
ExtraFiles []SourceFile
|
||||||
}
|
}
|
||||||
|
|
||||||
type DestinationStateOptions struct {
|
|
||||||
PipelineID string
|
|
||||||
DestinationID string
|
|
||||||
DistributorVersion string
|
|
||||||
PublishedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func DefaultSourceFiles() []SourceFile {
|
func DefaultSourceFiles() []SourceFile {
|
||||||
return []SourceFile{
|
return []SourceFile{
|
||||||
{Path: "report.md", Data: "# Report\nSunny.\n"},
|
{Path: "report.md", Data: "# Report\nSunny.\n"},
|
||||||
@@ -133,28 +124,6 @@ func AssertFakeMissing(t testing.TB, backend *fake.Backend, path string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func WriteFakeDestinationState(t testing.TB, backend *fake.Backend, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
|
||||||
t.Helper()
|
|
||||||
destinationState := DestinationState(manifest, opts)
|
|
||||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal destination state: %v", err)
|
|
||||||
}
|
|
||||||
statePath, err := storage.StatePath(relative)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("state path: %v", err)
|
|
||||||
}
|
|
||||||
WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
|
||||||
for _, output := range destinationState.Outputs {
|
|
||||||
path, err := storage.Join(relative, output.Path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("join output path: %v", err)
|
|
||||||
}
|
|
||||||
WriteFakeFile(t, backend, path, "old")
|
|
||||||
}
|
|
||||||
return destinationState
|
|
||||||
}
|
|
||||||
|
|
||||||
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
|
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
return writeConfigFile(t, `
|
return writeConfigFile(t, `
|
||||||
@@ -318,47 +287,6 @@ pipelines:
|
|||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func WriteDestinationState(t testing.TB, root, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
|
||||||
t.Helper()
|
|
||||||
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
|
|
||||||
if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
|
|
||||||
t.Fatalf("mkdir destination bundle: %v", err)
|
|
||||||
}
|
|
||||||
destinationState := DestinationState(manifest, opts)
|
|
||||||
writeJSONFile(t, filepath.Join(bundleRoot, storage.StateFileName), destinationState)
|
|
||||||
return destinationState
|
|
||||||
}
|
|
||||||
|
|
||||||
func DestinationState(manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
|
||||||
publishedAt := defaultPublishedAt(opts.PublishedAt)
|
|
||||||
return state.DistributorState{
|
|
||||||
SchemaVersion: state.SchemaVersion,
|
|
||||||
DistributorVersion: opts.DistributorVersion,
|
|
||||||
PipelineID: defaultString(opts.PipelineID, "reports"),
|
|
||||||
DestinationID: defaultString(opts.DestinationID, "archive"),
|
|
||||||
PublishedAt: publishedAt,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
|
||||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
||||||
Source: state.SourceState{Manifest: manifest},
|
|
||||||
Outputs: sourceOutputs(manifest, publishedAt),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ReadDestinationState(t testing.TB, path string) state.DistributorState {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read destination state: %v", err)
|
|
||||||
}
|
|
||||||
destinationState, err := state.Parse(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse destination state: %v", err)
|
|
||||||
}
|
|
||||||
return destinationState
|
|
||||||
}
|
|
||||||
|
|
||||||
func AssertFile(t testing.TB, path, want string) {
|
func AssertFile(t testing.TB, path, want string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
@@ -392,22 +320,6 @@ func sourceFiles(opts BundleOptions) []SourceFile {
|
|||||||
return files
|
return files
|
||||||
}
|
}
|
||||||
|
|
||||||
func sourceOutputs(manifest bundle.Manifest, publishedAt time.Time) []state.OutputFile {
|
|
||||||
outputs := make([]state.OutputFile, 0, len(manifest.Files))
|
|
||||||
for _, file := range manifest.Files {
|
|
||||||
outputs = append(outputs, state.OutputFile{
|
|
||||||
Path: file.Path,
|
|
||||||
Kind: state.OutputKindSource,
|
|
||||||
SourcePath: file.Path,
|
|
||||||
SHA256: file.SHA256,
|
|
||||||
Size: file.Size,
|
|
||||||
CreatedAt: publishedAt,
|
|
||||||
UpdatedAt: publishedAt,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return outputs
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeConfigFile(t testing.TB, body string) string {
|
func writeConfigFile(t testing.TB, body string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join(t.TempDir(), "config.yml")
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
@@ -455,10 +367,3 @@ func defaultCreated(value time.Time) time.Time {
|
|||||||
}
|
}
|
||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
func defaultPublishedAt(value time.Time) time.Time {
|
|
||||||
if value.IsZero() {
|
|
||||||
return time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
|
||||||
}
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user