24 Commits

Author SHA1 Message Date
8366af6fb6 Close completed catalog roadmap
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-06-19 17:15:24 +00:00
5e47d89355 Remove legacy publish and state paths 2026-06-19 17:12:21 +00:00
2e0d903626 Implement catalog idempotent publish skips 2026-06-19 16:57:38 +00:00
484fda2514 Synchronize catalog documentation and examples 2026-06-19 16:50:40 +00:00
9a2eaf8e5e Implement catalog force replacement 2026-06-19 16:39:01 +00:00
4e673dda76 Update the staged implementation plan to address remaining items from the destination policy refactor 2026-06-19 11:25:36 -05:00
cf7e62733e Support catalog maintenance commands 2026-06-19 15:58:36 +00:00
91b72478d5 Update run reporting for catalog workflows 2026-06-19 15:50:09 +00:00
52078e2195 Write catalog state during publish 2026-06-19 15:44:23 +00:00
77cde40296 Add catalog workflow planning 2026-06-19 15:36:26 +00:00
20129bfff5 Add catalog destination state schema 2026-06-19 15:28:03 +00:00
a95662226f Add destination workflow config 2026-06-19 15:18:14 +00:00
d23c624179 Added a roadmap and implementation plan for a significant refactor of pipeline destination policy and catalog state 2026-06-19 10:10:38 -05:00
69043801d0 Update policy for replacement of managed files 2026-06-18 13:38:44 -05:00
8b0ce4d134 Add takeover backend regressions and close roadmap 2026-06-18 15:40:24 +00:00
eba65018be Expose takeover actions in run output 2026-06-18 15:35:06 +00:00
11d1eabe2a Implement shared-root takeover policy 2026-06-18 15:27:50 +00:00
c02106987f Implement single-owner managed takeover 2026-06-18 15:18:38 +00:00
598b665307 Add structured destination comparison details 2026-06-18 15:09:18 +00:00
c4e8ebff6f Add destination takeover config policy 2026-06-18 15:05:09 +00:00
3da7f931b2 Add a feature roadmap and staged implementation plan for configurable levels of managed file overwrite protection 2026-06-18 10:01:20 -05:00
b10a8bd194 Add support for CSS links in generated HTML outputs 2026-06-13 22:16:15 -05:00
c84d8868d1 Fix a bug in the SFTP backend that would cause an error when overwriting existing files 2026-06-13 13:57:50 -05:00
fc33bbca54 Update documentation policy 2026-06-13 13:55:59 -05:00
66 changed files with 3595 additions and 5747 deletions

View File

@@ -72,7 +72,7 @@ distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
- `--config <path>` loads the pipeline configuration. If omitted, the application uses `/usr/local/etc/distributor/config.yml`.
- `--dry-run` validates inputs and reports destination actions without applying changes.
- `--force` permits a run when destination state indicates a conservative safety check would otherwise block it.
- `--force` permits exceptional catalog replacement when a dry run reports `force_replace` for unmanaged content, a planned unmanaged path collision, invalid state, or unsupported future state.
- `--format text|json` selects human-readable or machine-readable output.
`run` accepts no positional arguments.
@@ -86,7 +86,7 @@ distributor reconcile-state --config <path> --pipeline <id> --destination <id> [
- `--config <path>` loads the pipeline configuration and is required.
- `--pipeline <id>` selects the pipeline used to identify the destination root and is required.
- `--destination <id>` selects the destination root and is required.
- `--all-owners` repairs missing managed output records for every owner in a shared-root state file. Without it, shared-root repair is scoped to the selected pipeline and destination owner.
- `--all-owners` repairs missing managed output records for every owner in the selected catalog state file. Without it, repair is scoped to the selected pipeline and destination owner.
- `--dry-run` reports repairs without rewriting `.distributor.json`.
- `--format text|json` selects human-readable or machine-readable output.
@@ -206,7 +206,7 @@ go run ./cmd/distributor run --config examples/local-publish.yml --dry-run
go run ./cmd/distributor run --config examples/local-publish.yml
```
Use `--format json` when automation needs structured run results. Use `--force` only when the operator has reviewed the destination state conflict and intentionally wants to continue.
Use `--format json` when automation needs structured run results. Use `--force` only after `--dry-run --force` reports the intended bounded `force_replace` action.
### Repair Destination State Records
@@ -229,7 +229,7 @@ go run ./cmd/distributor reconcile-state \
--destination local-archive
```
Use `--all-owners` only for shared-root destination state when all owners inside the selected root should be repaired.
Use `--all-owners` only when every owner inside the selected catalog root should be repaired.
### Prune Managed Outputs
@@ -292,6 +292,8 @@ Text output is optimized for direct operator use. JSON output is optimized for a
- Usage errors and fatal setup errors exit non-zero and do not emit a JSON result document.
- `run --format json` emits a JSON result for partial destination failures, sets `ok` to `false`, includes result details and errors, and exits non-zero.
- Warnings are included in JSON output and are printed in text output when relevant.
- `run` summaries include `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and `fail_conflict` counters. Destination action records use the same stable action values and include the resolved `destination_path`.
- `skip_same` means the planned outputs already match valid catalog metadata, so `run` does not write outputs, rewrite `.distributor.json`, delete files, or notify. The decision is based on catalog metadata and does not read destination file bytes.
## Diagnostics And Recovery

View File

@@ -2,11 +2,11 @@
Audience: administrators, operators, and advanced users who write YAML configuration for `distributor`.
This document is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operating procedures live in [Operations](operations.md), symptom-oriented recovery lives in [Troubleshooting](troubleshooting.md), and external contracts live under [Integrations](integrations/source-bundle.md).
This is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operations guidance lives in [Operations](operations.md), recovery guidance lives in [Troubleshooting](troubleshooting.md), and file-format contracts live under [Integrations](integrations/source-bundle.md).
## Config File Loading
`distributor run --config <path>` and `distributor serve --config <path>` load the YAML file at `<path>`. If `--config` is omitted, both commands use:
`distributor run --config <path>` and `distributor serve --config <path>` load the YAML file at `<path>`. If `--config` is omitted, commands use:
```text
/usr/local/etc/distributor/config.yml
@@ -21,7 +21,7 @@ Runtime backend support is command-specific:
- `serve` uses `http_upload` sources through the HTTP upload API and publishes to configured `local`, `ssh`, and `s3` destinations.
- `http_upload` is valid only as a source backend.
## Minimal Local Config
## Minimal Working Config
```yaml
pipelines:
@@ -35,9 +35,9 @@ pipelines:
path: /srv/reports/archive
```
This config publishes source files only. It uses default validation, destination path mapping, publish, state, reconciliation, retention, transfer, and HTTP server values.
This publishes source files only. It uses default validation, additive workflow, preserve-relative path mapping, source-only publish policy, disabled pruning, and default HTTP server values.
## Production-Oriented Local Config
## Production-Oriented Config
```yaml
server:
@@ -61,23 +61,20 @@ pipelines:
- id: archive
backend: local
path: /srv/reports/archive
workflow: additive
publish:
source: true
html: false
path_mapping:
mode: preserve_relative
reconciliation:
mode: replace
transfer:
on_destination_same: skip
on_destination_older: replace
on_destination_newer: skip
on_conflict: fail
retention:
prune:
enabled: false
```
## HTTP Upload Source Config
HTTP upload sources are configured on pipelines and are served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML.
HTTP upload sources are configured on pipelines and served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML.
```yaml
server:
@@ -97,15 +94,13 @@ pipelines:
- id: weather-daily
source:
backend: http_upload
staging_path: /var/spool/distributor/weather-daily
max_upload_size: 20MB
destinations:
- id: archive
backend: local
path: /srv/reports/archive
```
`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to. One token may authorize multiple upload pipelines, and multiple tokens may authorize the same upload pipeline.
`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to.
For `http_upload` sources, `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`. `max_upload_size` defaults to `server.http.max_upload_size`.
@@ -136,7 +131,7 @@ See [Secrets](#secrets) for resolution rules.
Each token has:
- `id`: required unique slug-like identifier for the token record. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
- `id`: required unique slug-like identifier for the token record.
- `token_env`: required environment variable or secret-file name containing the bearer token value.
- `allow_pipelines`: required non-empty list of configured pipeline ids whose source backend is `http_upload`.
@@ -148,12 +143,12 @@ Token values must resolve to non-empty strings and must be unique across token r
Each pipeline has:
- `id`: required unique slug-like identifier. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
- `id`: required unique slug-like identifier.
- `source`: required source backend config.
- `validation`: optional validation policy.
- `destinations`: required non-empty destination list.
Pipeline ids must be unique across the config.
Slug-like identifiers must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
## Backend Reference
@@ -171,7 +166,7 @@ path: /srv/distributor/archive
### SSH/SFTP Backend
SSH backends use native SFTP and can be used as sources and destinations. Adapter protocol behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md).
SSH backends use native SFTP and can be used as sources and destinations. Adapter behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md).
```yaml
backend: ssh
@@ -193,17 +188,11 @@ host_key_policy: strict
- `known_hosts`: optional OpenSSH `known_hosts` path.
- `host_key_policy`: optional host key policy. Default: `accept-new`.
Accepted host key policy values:
- `strict` or boolean `true`: require a matching known host key.
- `accept-new`: accept and persist a new host key, but reject changed known keys.
- `off` or boolean `false`: disable host key checking.
Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML.
Accepted host key policy values are `strict` or boolean `true`, `accept-new`, and `off` or boolean `false`. Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML.
### S3-Compatible Backend
S3 backends can be used as sources and destinations. Adapter protocol behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md).
S3 backends can be used as sources and destinations. Adapter behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md).
```yaml
backend: s3
@@ -255,42 +244,50 @@ Source bundle digest mismatches fail validation before destination writes occur.
## Destination Fields
Each destination embeds a backend config at the destination level and may also configure publishing, transforms, path mapping, links, state, reconciliation, retention, and transfer behavior.
Each destination embeds a backend config at the destination level and may also configure workflow, publishing, transforms, path mapping, links, and retention.
```yaml
destinations:
- id: archive
backend: local
path: /srv/reports/archive
workflow: additive
publish:
source: true
html: false
path_mapping:
mode: preserve_relative
state:
mode: single_owner
reconciliation:
mode: replace
transfer:
on_destination_same: skip
on_destination_older: replace
on_destination_newer: skip
on_conflict: fail
```
- `id`: required unique slug-like identifier within the pipeline.
- Backend fields: required according to the selected destination backend.
- `workflow`: optional catalog update workflow. Default: `additive`.
- `publish`: optional publish policy. Default: source-only publication.
- `transform`: required only when publishing generated HTML.
- `path_mapping`: optional destination path mapping policy.
- `links`: optional public URL metadata policy.
- `state`: optional destination state ownership policy.
- `reconciliation`: optional managed-output reconciliation policy.
- `retention`: optional managed-output retention policy.
- `transfer`: optional destination comparison action policy.
Destination ids must be unique within a pipeline.
Pre-workflow destination policy keys for state mode, conflict handling,
ownership adoption, and per-comparison copy decisions are not accepted config
fields. YAML files containing those keys fail during config loading.
## Destination Workflow
```yaml
workflow: additive
```
- `workflow`: optional. Accepted values are `additive` and `replacement`; default is `additive`.
`additive` writes planned outputs into the catalog and retains unrelated catalog-managed outputs in the same destination bundle path. Existing catalog records for planned paths are replaced by the current publication. A planned path that exists in storage but is not recorded in valid catalog state fails as unmanaged content unless `run --force` selects `force_replace`.
`replacement` writes planned outputs for the current pipeline and destination, and removes catalog outputs owned by the same pipeline and destination when those outputs are omitted from the new plan. Outputs owned by other pipeline/destination pairs remain catalog-managed. Replacement workflow is normal managed behavior and does not require `--force`.
Use `replacement` for stable latest-style destinations where the current owner should publish exactly the currently planned output set. Use `additive` when a destination root intentionally accumulates outputs over time or receives disjoint outputs from multiple configured destinations.
## Publish And Transform Policy
### Source-Only Publication
@@ -313,6 +310,7 @@ transform:
markdown_to_html:
enabled: true
mode: sidecar
css_href: /assets/report.css
```
`publish.html` controls whether generated HTML outputs are published. When `publish.html` is `true`, `transform.markdown_to_html.enabled` must also be `true`.
@@ -322,10 +320,13 @@ Markdown transform fields:
- `transform.markdown_to_html.enabled`: enables Markdown-to-HTML generation for this destination.
- `transform.markdown_to_html.mode`: optional. Accepted values are `sidecar` and `index`; default is `sidecar` when a Markdown transform block is present.
- `transform.markdown_to_html.input`: optional source manifest path for `index` mode only.
- `transform.markdown_to_html.css_href`: optional stylesheet href to link from generated HTML.
`sidecar` mode renders every manifest-listed `.md` file to a same-directory `.html` output. `index` mode renders one Markdown source to `index.html` at the destination bundle path. If `index` mode omits `input`, the selected source bundle must contain exactly one Markdown file.
At least one output type must be enabled. Enabled Markdown transforms are rejected when `publish.html` is `false`, and `input` is rejected unless `mode` is `index`.
`css_href` may be an absolute `http` or `https` URL, a root-relative path such as `/assets/report.css`, or a relative URL path such as `assets/report.css`. Query strings are allowed. `distributor` injects the href as a `<link rel="stylesheet">` element but does not copy, publish, verify, or manage the CSS file solely because `css_href` is set.
At least one output type must be enabled. Enabled Markdown transforms are rejected when `publish.html` is `false`, `input` is rejected unless `mode` is `index`, and `css_href` is rejected when the Markdown transform is disabled.
## Destination Path Mapping
@@ -361,93 +362,7 @@ Primary URL policies:
- `html`: use the first generated HTML output.
- `source`: use the first copied source output.
If no output matches the primary policy, per-output URLs may still be recorded and the top-level primary URL is omitted.
## Destination State Policy
```yaml
state:
mode: single_owner
```
- `state.mode`: optional. Accepted values are `single_owner` and `shared_root`; default is `single_owner`.
`single_owner` state records one pipeline/destination owner for each destination bundle path and is the default state mode.
`shared_root` records multiple pipeline/destination owners in one destination root. Publish execution preserves unrelated owners, rejects path ownership conflicts, and writes shared-root destination state.
Use `shared_root` when multiple configured destinations intentionally write disjoint output paths into the same backend root:
```yaml
pipelines:
- id: reports-source
source:
backend: local
path: /var/spool/distributor/reports
destinations:
- id: shared-root
backend: local
path: /srv/reports/shared
state:
mode: shared_root
publish:
source: true
html: false
- id: reports-html
source:
backend: local
path: /var/spool/distributor/reports
destinations:
- id: shared-root
backend: local
path: /srv/reports/shared
state:
mode: shared_root
publish:
source: false
html: true
transform:
markdown_to_html:
enabled: true
mode: sidecar
```
Every output path in a shared root belongs to exactly one `pipeline_id` and `destination_id`. A different owner planning the same path fails as a conflict.
## Reconciliation Policy
```yaml
reconciliation:
mode: replace
```
- `reconciliation.mode`: optional. Accepted values are `replace` and `merge`; default is `replace`.
Reconciliation controls how a destination with older managed state is updated after transfer policy selects `replace_older`.
`replace` deletes the prior managed outputs recorded in `.distributor.json`, deletes the state file, verifies the destination bundle path is empty, then writes only the newly planned outputs and a new state file. This is the default and is appropriate when each publication should exactly match the current publish and transform policy.
`merge` keeps prior managed outputs that are not produced by the new plan. Planned paths already recorded in existing state may be overwritten; planned paths that already exist in storage but are not recorded as managed fail as unmanaged collisions. The resulting state file records the cumulative managed output set.
Example merge destination:
```yaml
destinations:
- id: static-site
backend: local
path: /srv/reports/site
publish:
source: false
html: true
transform:
markdown_to_html:
enabled: true
mode: sidecar
reconciliation:
mode: merge
```
`links.primary_url` is selected from the newly planned outputs for the current run. Retained outputs keep their prior output metadata and timestamps.
If no output matches the primary policy, per-output URLs may still be recorded and no primary URL is reported for the run.
## Retention Policy
@@ -465,28 +380,9 @@ retention:
When `retention.prune.enabled` is `true`, at least one of `older_than` or `keep_latest` is required. `older_than` must be greater than zero, and `keep_latest` must be zero or greater.
Pruning uses managed output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs.
Pruning uses catalog output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs.
The `prune` command is owner-scoped for shared-root state. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`.
## Transfer Policy
```yaml
transfer:
on_destination_same: skip
on_destination_older: replace
on_destination_newer: skip
on_conflict: fail
```
Transfer fields and accepted values:
- `transfer.on_destination_same`: `skip` or `fail`. Default: `skip`.
- `transfer.on_destination_older`: `replace` or `fail`. Default: `replace`.
- `transfer.on_destination_newer`: `skip`, `replace`, or `fail`. Default: `skip`.
- `transfer.on_conflict`: `fail` or `replace`. Default: `fail`.
`replace` for `on_destination_newer` and `on_conflict` is honored only when `run --force` is supplied. There is no config field that enables forced replacement by default.
The `prune` command is scoped to the selected pipeline and destination owner. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`.
## Size And Duration Values
@@ -519,17 +415,12 @@ Defaults are applied after YAML decoding and before validation:
- S3 `force_path_style: true`
- `http_upload` source `staging_path: <server.http.staging_root>/<pipeline id>`
- `http_upload` source `max_upload_size: server.http.max_upload_size`
- `workflow: additive`
- `publish.source: true` and `publish.html: false`
- `transform.markdown_to_html.mode: sidecar` when a Markdown transform block is present and mode is omitted
- `path_mapping.mode: preserve_relative`
- `links.primary: auto` when a `links` block is present and `primary` is omitted
- `state.mode: single_owner`
- `reconciliation.mode: replace`
- `retention.prune.enabled: false`
- `transfer.on_destination_same: skip`
- `transfer.on_destination_older: replace`
- `transfer.on_destination_newer: skip`
- `transfer.on_conflict: fail`
## Secrets
@@ -552,7 +443,7 @@ Fields resolved through this resolver:
## Maintained Examples
Maintained examples live under [examples](../examples/). Config tests load every file listed here.
Maintained examples live under [examples](../examples/). Config tests load these YAML files.
Local examples:
@@ -562,8 +453,8 @@ Local examples:
- `local-index.yml`: local `index.html` publication.
- `fan-out.yml`: local fan-out publication to source and HTML destinations.
- `archive-and-latest.yml`: local archive plus fixed latest publication.
- `merge-reconciliation.yml`: local HTML publication using merge reconciliation.
- `shared-root.yml`: two local pipelines publishing disjoint outputs into one shared destination root.
- `additive-workflow.yml`: two destinations publishing disjoint outputs into one catalog-managed root.
- `replacement-workflow.yml`: fixed-path replacement workflow for a stable latest-style output set.
- `http-upload-local.yml`: local HTTP upload server config; requires `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` in the process environment or as a secret-file name before running `serve`.
Environment-gated remote examples:

View File

@@ -2,194 +2,37 @@
Audience: operators, integrators, and maintainers who inspect or reason about destination `.distributor.json` files.
Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for comparison, skip, replacement, and recovery decisions.
Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for catalog planning, managed replacement, retention pruning, repair, and recovery.
## Single-Owner State Schema
## Catalog State Schema
State written by `run` for `state.mode: single_owner` uses schema version `2`.
Publish execution writes catalog state with `schema_version` `4`.
```json
{
"schema_version": 2,
"distributor_version": "dev",
"pipeline_id": "reports",
"destination_id": "archive",
"published_at": "2026-06-04T12:00:00Z",
"created_at": "2026-06-04T12:00:00Z",
"updated_at": "2026-06-04T12:00:00Z",
"state": {
"mode": "single_owner"
},
"reconciliation": {
"mode": "replace"
},
"source": {
"manifest": {
"schema_version": 1,
"id": "reports.example.2026-06-04",
"digest": "sha256:...",
"created": "2026-06-04T11:55:00Z",
"files": [
{"path": "report.md", "sha256": "sha256:...", "size": 1234}
]
}
},
"links": {
"primary_url": "https://reports.example.com/archive/report.html"
},
"outputs": [
{
"path": "report.html",
"kind": "generated",
"source_path": "report.md",
"transform": "markdown_to_html",
"url": "https://reports.example.com/archive/report.html",
"sha256": "sha256:...",
"size": 2345,
"created_at": "2026-06-04T12:00:00Z",
"updated_at": "2026-06-04T12:00:00Z"
}
]
}
```
Required fields:
- `schema_version`: must be `2` for newly written state.
- `pipeline_id`: configured pipeline id that wrote the state.
- `destination_id`: configured destination id that wrote the state.
- `published_at`: RFC3339 publication timestamp.
- `created_at`: RFC3339 timestamp for when this destination state record was first created.
- `updated_at`: RFC3339 timestamp for the latest state update.
- `state.mode`: must be `single_owner`.
- `reconciliation.mode`: `replace` or `merge`.
- `source.manifest`: embedded source bundle manifest.
- `outputs`: output records array; empty is allowed, but the field is required.
Optional fields:
- `distributor_version`: application version string when available.
- `links.primary_url`: absolute HTTP or HTTPS URL selected by destination link policy.
## Output Records
Each output record has:
- `path`: destination-relative output path.
- `kind`: `source` or `generated`.
- `source_path`: source manifest path used for the output.
- `transform`: required for `generated` outputs; omitted for copied source outputs.
- `url`: optional absolute HTTP or HTTPS URL for the output.
- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes.
- `size`: output byte size, zero or greater.
- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed.
- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state.
Output paths must be unique and use clean relative slash-separated path rules.
For replacement updates, newly planned outputs are written into state. For merge updates, retained output records preserve both timestamps, overwritten managed output records preserve `created_at` and receive a new `updated_at`, and new output records receive the current publication time for both fields.
## Reconciliation Semantics
Destination reconciliation applies when destination state is older than the source and transfer policy permits replacement.
- `replace`: for single-owner state, delete managed output paths recorded in `outputs` plus `.distributor.json`, require the destination bundle path to be empty afterward, write the newly planned outputs, and write state whose `outputs` are exactly that new planned set. For shared-root state, delete only the current owner's omitted outputs and preserve unrelated owners.
- `merge`: retain managed output paths omitted from the new plan, overwrite planned paths only when they are already recorded in existing state, fail when a newly planned path exists in storage but is not recorded as managed, and write state whose `outputs` are the cumulative managed set.
Top-level `links.primary_url` is selected from the newly planned outputs for the current publication. Retained outputs keep their existing per-output URL metadata.
If a merge publication fails after writing outputs, cleanup removes only newly created outputs from that failed attempt. Previously managed overwritten paths remain managed and are not removed by failed-attempt cleanup.
## Comparison Semantics
`distributor` compares the current source manifest to destination state before writing:
- No state and no content: publish new outputs.
- No state and existing content: treat the destination as unmanaged.
- Shared-root state without the current owner: publish new outputs for that owner if planned paths do not collide with other owners or unmanaged content.
- Matching embedded source manifest: skip.
- Same source id with older `created`: replace if policy allows.
- Same source id with newer `created`: skip by default.
- Same source id and same `created` with different digest: conflict.
- Different source id, pipeline id, or destination id: conflict.
- Invalid state JSON or invalid state fields: conflict.
Normal single-owner replacement deletes only managed output paths recorded in `outputs` plus `.distributor.json`. Shared-root replacement deletes only omitted outputs for the current owner. Merge publication retains omitted managed outputs. Forced replacement deletes the bounded destination bundle path.
## State Repair Semantics
`distributor reconcile-state` can remove managed output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. The command checks output paths recorded in `outputs`, reports missing managed outputs, reports unmanaged entries under the destination root, and removes missing output records from valid state unless `--dry-run` is set.
For shared-root state, repair is scoped to the selected owner by default. With `--all-owners`, it checks and repairs missing output records for every owner in the selected shared-root state file.
State repair does not validate output digests, delete destination files, adopt unmanaged entries, or rewrite invalid or mismatched state.
## Prune Semantics
`distributor prune` can delete managed outputs selected by the configured destination retention policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
Prune planning uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`.
## Compatibility
`distributor` can read schema version `1` destination state for compatibility. When v1 state is read, it is treated as single-owner state with `reconciliation.mode: replace`. Missing top-level `created_at` and `updated_at` are inferred from `published_at`, and missing per-output timestamps are also inferred from `published_at`.
Newly written single-owner destination state from publish execution uses schema version `2`.
## Shared-Root State Schema
State written by `run` for `state.mode: shared_root` uses schema version `3`.
```json
{
"schema_version": 3,
"schema_version": 4,
"distributor_version": "dev",
"created_at": "2026-06-04T12:00:00Z",
"updated_at": "2026-06-04T12:10:00Z",
"state": {
"mode": "shared_root"
"mode": "catalog"
},
"owners": [
{
"pipeline_id": "reports",
"destination_id": "archive",
"reconciliation": {
"mode": "merge"
},
"source": {
"manifest": {
"schema_version": 1,
"id": "reports.example.2026-06-04",
"digest": "sha256:...",
"created": "2026-06-04T11:55:00Z",
"files": [
{"path": "report.md", "sha256": "sha256:...", "size": 1234}
]
}
},
"links": {
"primary_url": "https://reports.example.com/archive/report.html"
}
}
],
"outputs": [
{
"path": "report.html",
"pipeline_id": "reports",
"destination_id": "static-site",
"source": {
"id": "reports.example.2026-06-04",
"digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
"created": "2026-06-04T11:55:00Z"
},
"kind": "generated",
"source_path": "report.md",
"transform": "markdown_to_html",
"url": "https://reports.example.com/archive/report.html",
"sha256": "sha256:...",
"sha256": "sha256:abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd",
"size": 2345,
"pipeline_id": "reports",
"destination_id": "archive",
"source_id": "reports.example.2026-06-04",
"source_digest": "sha256:...",
"source_created": "2026-06-04T11:55:00Z",
"created_at": "2026-06-04T12:00:00Z",
"updated_at": "2026-06-04T12:10:00Z"
}
@@ -197,55 +40,88 @@ State written by `run` for `state.mode: shared_root` uses schema version `3`.
}
```
Shared-root required fields:
Top-level fields:
- `schema_version`: must be `3`.
- `created_at`: RFC3339 timestamp for when this shared-root state record was first created.
- `updated_at`: RFC3339 timestamp for the latest shared-root state update.
- `state.mode`: must be `shared_root`.
- `owners`: owner records keyed by `pipeline_id` and `destination_id`; each owner records its latest source manifest and reconciliation mode.
- `outputs`: output records for every managed path under the shared destination root.
- `schema_version`: required value `4`.
- `distributor_version`: optional application version string.
- `created_at`: RFC3339 timestamp for when this catalog record was first created.
- `updated_at`: RFC3339 timestamp for the latest catalog update.
- `state.mode`: required value `catalog`.
- `outputs`: required array of catalog output records. Empty is valid.
Shared-root optional fields:
## Output Records
- `distributor_version`: application version string when available.
- `owners[].links.primary_url`: absolute HTTP or HTTPS URL selected by that owner's destination link policy.
Each output record has:
Shared-root output records carry the same `path`, `kind`, `source_path`, `transform`, `url`, `sha256`, `size`, `created_at`, and `updated_at` fields as single-owner outputs. They also include the owner `pipeline_id` and `destination_id`, plus compact source identity fields `source_id`, `source_digest`, and `source_created`.
- `path`: destination-bundle-relative output path.
- `pipeline_id`: configured pipeline id that manages the output path.
- `destination_id`: configured destination id that manages the output path.
- `source`: compact source identity for the output.
- `kind`: `source` or `generated`.
- `source_path`: generated outputs only; source manifest path used to derive the output.
- `transform`: generated outputs only; transform id, currently `markdown_to_html`.
- `url`: optional absolute HTTP or HTTPS URL for the output.
- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes.
- `size`: output byte size, zero or greater.
- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed.
- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state.
## Shared-Root Ownership
Output paths must be unique and use clean relative slash-separated path rules. `pipeline_id` and `destination_id` must be slug-like identifiers.
Shared-root state is owner-scoped by `pipeline_id` and `destination_id`.
For copied source outputs, `source_path` and `transform` are omitted. For generated outputs, both fields are required.
- One output path may be owned by only one owner.
- The same owner may overwrite its own managed paths.
- A different owner planning an already owned path fails as a conflict.
- A planned path that exists in storage but is not recorded in state fails as unmanaged content unless forced replacement is explicitly selected.
## Source Identity
When an owner publishes, unrelated owner records and output records are preserved. The publishing owner's record is updated with the latest source manifest, reconciliation mode, and latest primary URL when present.
Each output records the source identity that produced it:
## Shared-Root Timestamps
- `source.id`: source manifest id.
- `source.digest`: source manifest digest.
- `source.created`: source manifest creation timestamp.
For shared-root state:
The full source manifest is not embedded in catalog state. The source bundle's `manifest.json` remains the producer contract, and `.distributor.json` records only the source identity needed for catalog ownership and later maintenance workflows.
- top-level `created_at` remains the original shared-root state creation time;
- top-level `updated_at` changes after a successful state write;
- output `created_at` remains stable for an existing managed path;
- output `updated_at` changes only when that path is rewritten;
- newly managed output paths receive the publication time for both output timestamps.
## Workflow Semantics
## Shared-Root Migration
Destination `workflow` is runtime configuration and is not persisted in `.distributor.json`.
If `state.mode: shared_root` is configured and existing state is a compatible single-owner `.distributor.json` for the same pipeline id and destination id, the next successful publish writes schema version `3` shared-root state for that owner.
`workflow: additive` writes planned outputs and retains unrelated catalog-managed outputs. If a planned path already has a catalog output record, the current publication replaces that record and overwrites the file. If a planned path exists in storage but is not recorded in valid catalog state, planning fails as unmanaged unless `run --force` selects `force_replace`.
If existing single-owner state belongs to a different pipeline or destination, publish fails as a conflict. `distributor` does not implicitly take over unrelated state or unmanaged files.
`workflow: replacement` writes planned outputs for the current pipeline and destination and removes omitted outputs owned by that same pipeline and destination. Outputs owned by other pipeline/destination pairs remain catalog-managed. This is normal managed replacement and does not require `--force`.
`force_replace` is an explicit per-run recovery path. It deletes only the resolved destination bundle path, then writes planned outputs and fresh catalog state. It can replace unmanaged content, planned unmanaged path collisions, invalid destination state, and unsupported future destination state after dry-run review.
## Publish Planning Outcomes
Current run reports use these destination action labels:
- `publish_new`: no valid state exists and the destination bundle path is empty.
- `upsert_additive`: valid catalog state exists and additive workflow will write the planned outputs.
- `replace_catalog`: valid catalog state exists and replacement workflow will write the planned outputs and remove omitted outputs for the current owner.
- `skip_same`: no-op action value in the run output vocabulary.
- `force_replace`: explicit bounded destructive replacement selected by `--force`.
- `fail_unmanaged`: unmanaged destination content prevents publication.
- `fail_conflict`: invalid state or unsupported state prevents publication without explicit force.
Schema versions older than `4` are superseded legacy state for publish planning. Normal catalog planning may publish over superseded legacy state according to the configured workflow, while invalid state and unsupported future schema versions fail unless `--force` is explicitly selected.
## Repair Semantics
`distributor reconcile-state` removes catalog output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
By default, repair is scoped to output records whose `pipeline_id` and `destination_id` match the selected pipeline and destination. With `--all-owners`, it checks every catalog output record in the selected root.
The command reports missing managed outputs and unmanaged storage entries. Without `--dry-run`, it removes missing managed output records from valid catalog state and rewrites `.distributor.json`. It does not delete destination files, adopt unmanaged entries, validate output digests, or rewrite invalid state.
## Prune Semantics
`distributor prune` deletes catalog output paths selected by the configured destination `retention.prune` policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
Prune planning is scoped to output records whose `pipeline_id` and `destination_id` match the selected pipeline and destination. It uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid catalog state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`.
## Boundaries
Destination state is internal managed state written by `distributor`. Operators may inspect it during recovery, but normal workflows should not edit it by hand. Source `manifest.json` is not copied as destination state.
## Tests
Before changing this contract, inspect and run:
```sh

View File

@@ -10,7 +10,7 @@ Rendering uses `github.com/yuin/goldmark`. The exact version is pinned in `go.mo
## Renderer Behavior
The transformer constructs `goldmark.New()` with no project-specific extensions, parser options, renderer options, templates, CSS, or metadata injection.
The transformer constructs `goldmark.New()` with no project-specific extensions, parser options, renderer options, templates, or source manifest metadata injection.
Supported output modes:
@@ -19,6 +19,8 @@ Supported output modes:
In `index` mode, `transform.markdown_to_html.input` may name the source manifest path to render. If `input` is omitted, the source manifest must list exactly one `.md` file. The selected input must be a clean relative source path, must be listed in the source manifest, and must end in `.md`.
When `transform.markdown_to_html.css_href` is set, generated HTML includes a stylesheet link in the document head. The href may be an absolute HTTP(S) URL, a root-relative path, or a relative URL path. Distributor treats this as a link reference only; it does not copy, publish, verify, or manage the CSS file solely because `css_href` is configured.
Raw HTML embedded in Markdown is not passed through by the current renderer behavior. Tests allow Goldmark's disabled-or-escaped raw HTML output forms and reject literal script tags in generated HTML.
## HTML Wrapper
@@ -28,10 +30,11 @@ Rendered Markdown body HTML is wrapped in a fixed document shell:
- `<!doctype html>`
- `<html lang="en">`
- UTF-8 `<meta charset>`
- optional `<link rel="stylesheet" href="...">` when `css_href` is configured
- empty `<title>`
- `<body>` containing the rendered Markdown body
The wrapper is deterministic and does not read configuration, templates, CSS, or source manifest metadata.
The wrapper is deterministic. When `css_href` is omitted, the generated wrapper is unchanged from the unstyled output. When `css_href` is configured, its escaped link element is part of the generated output bytes.
## Output Metadata
@@ -47,7 +50,7 @@ Destination state stores generated outputs with `kind: generated`, `source_path`
## Boundaries
Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose transfer policy. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata.
Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose catalog workflow behavior. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata.
## Tests

View File

@@ -52,7 +52,7 @@ The adapter uses these S3 operations:
Writes buffer the input and set `ContentLength`. If no content type is supplied by the caller, the adapter infers a content type from the logical path.
Normal replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Merge publication retains omitted managed objects and may overwrite existing managed objects. Forced replacement deletes objects under the bounded destination bundle prefix. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
Normal managed replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Forced replacement deletes objects under the bounded destination bundle prefix, then writes planned outputs and schema version `4` catalog state. For fixed-path destinations, that bounded prefix is the configured backend root. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
## Error Mapping

View File

@@ -54,7 +54,7 @@ The configured `path` is the backend root. All source discovery, destination pat
The adapter rejects symlink ancestors for reads and writes. Reads require regular files. Writes create parent directories and prefer atomic temp-file-plus-rename writes when requested. Walk output is sorted through the shared storage walker.
Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Merge publication retains omitted managed paths and may overwrite existing managed paths. Forced replacement deletes the bounded destination bundle path.
Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Forced replacement deletes the bounded destination bundle path, then writes planned outputs and schema version `4` catalog state. For fixed-path destinations, that bounded path is the configured backend root.
## Boundaries

View File

@@ -20,7 +20,7 @@ User-facing command parsing stays in `internal/cli`, including `reconcile-state`
## Config Fields Used
The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, retention policy, and transfer policy.
The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, workflow, publish policy, transform policy, path mapping, links, and retention policy.
Config fields are validated and defaulted by `internal/config` before app workflows use them.
@@ -32,11 +32,11 @@ The app layer registers default transforms, including Markdown-to-HTML, and supp
## State And Manifest Behavior
Run workflows discover and validate source bundles through `internal/bundle`. Destination state actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results.
Run workflows discover and validate source bundles through `internal/bundle`. Destination catalog actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results. Run summaries count `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and `fail_conflict` separately.
Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing managed output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from state and rewrites valid state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid or mismatched state.
Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing catalog output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from valid catalog state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid state.
Prune planning consumes a parsed destination state document and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state.
Prune planning consumes parsed catalog state and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state.
Prune execution loads one configured pipeline/destination selector, opens that destination root, parses the root `.distributor.json`, and builds a plan from the destination retention policy. Dry-run returns the same planned and preserved managed output records without deleting files or rewriting state. Apply mode deletes only planned managed output paths, never unmanaged files or `.distributor.json`, then removes confirmed deleted records from state and updates the state timestamp. If a delete fails after earlier deletes succeeded, it rewrites state only for the confirmed deletions and preserves records for the failed and unattempted outputs so a retry remains accurate. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope.
@@ -54,9 +54,9 @@ HTTP upload status is in memory. Accepted jobs move through accepted, queued, ru
Runtime setup fails for config load, config validation, secret loading, or credential resolution errors. Source setup failures stop the affected run before destination planning. Destination open, planning, execution, and notification failures are recorded as destination failures where a partial result exists.
Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root all-owner repair still uses the selected destination to identify the root, then applies repair across owners inside that root. Invalid, unreadable, or ambiguous state fails before any rewrite.
Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Catalog repair is scoped to that owner unless `--all-owners` is set. Invalid or unreadable state fails before any rewrite.
Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root pruning is scoped to the selected owner and preserves unrelated owners. Invalid, unreadable, or ambiguous state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output.
Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Pruning is scoped to the selected owner and preserves unrelated owners. Invalid or unreadable state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output.
HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Upload requests can fail during authentication, idempotency-key validation, content-type validation, idempotency conflict checks, queue admission, archive staging, source validation, or later publish execution.

View File

@@ -18,7 +18,7 @@ The canonical user-facing config reference is `docs/config.md`.
## Config Fields Used
The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, retention policy, and transfer policy.
The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, destination workflow, publish policy, transform policy, path mapping, links, and retention policy.
## Adapters Used
@@ -26,11 +26,11 @@ No external storage adapters are used directly. The package exposes normalized c
## State And Manifest Behavior
The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as publish/transform combinations, links, state policy, reconciliation policy, retention policy, transfer policy, backend roots, S3 prefix shape, and HTTP upload source settings.
The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as workflow, publish/transform combinations, links, retention policy, backend roots, S3 prefix shape, and HTTP upload source settings. Legacy destination policy YAML fields are rejected by strict decoding because they are not user-visible config fields.
## Skip And Resume Behavior
The package has no runtime skip or resume behavior. It provides transfer policy values that publish planning later applies to destination comparison outcomes.
The package has no runtime skip or resume behavior. Publish planning later applies workflow values and per-run force options to destination catalog outcomes.
## Failure Behavior

View File

@@ -4,13 +4,13 @@ Audience: developers and LLM coding agents changing `internal/publish`.
## Purpose
`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns destination comparison mapping, output selection, URL planning, managed cleanup selection, replacement safety, and destination state projection.
`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns output selection, URL planning, catalog action selection, managed cleanup selection, forced replacement safety, and destination state projection.
## Inputs And Outputs
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, state policy, reconciliation policy, transformer resolver, transfer policy, distributor version, and force flag.
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, workflow, transformer resolver, distributor version, force flag, and request time.
Output from planning is a `Plan` with action, reason, destination identity, selected outputs, state mode, owner scope, reconciliation mode, optional existing single-owner or shared-root state, optional primary URL, and force metadata. Shared-root plans also expose other-owner outputs to preserve, current-owner outputs retained by merge, current-owner outputs deleted by replace, and current-owner outputs to write. Execution writes selected source outputs, generated outputs, and `.distributor.json` for executable publish or replacement actions.
Output from planning is a `Plan` with action, reason, destination identity, selected outputs, workflow, owner scope, optional existing catalog state, optional superseded legacy marker, optional primary URL, catalog outputs to write, catalog outputs to retain, catalog outputs to delete, force metadata, and clear-root metadata. Execution writes selected source outputs, generated outputs, and schema version `4` `.distributor.json` for executable catalog actions.
## Boundaries
@@ -20,7 +20,7 @@ External destination state semantics are documented in `docs/integrations/destin
## Config Fields Used
The package consumes already-defaulted config values for destination `publish`, `transform`, `links`, `state`, `reconciliation`, `transfer`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
The package consumes already-defaulted config values for destination `workflow`, `publish`, `transform`, `links`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
## Adapters Used
@@ -28,27 +28,37 @@ The package depends on `internal/storage.Backend` for source and destination IO,
## State And Manifest Behavior
Planning inspects destination state through `internal/state`, compares it with the source manifest, and maps comparison outcomes plus transfer policy into actions: `publish_new`, `replace_older`, `force_replace`, `skip_same`, `skip_destination_newer`, `fail_conflict`, or `fail_unmanaged`.
Planning inspects destination state through `internal/state` and maps catalog conditions into actions:
Single-owner destinations compare the whole destination state against the configured pipeline and destination ids. Shared-root destinations compare only the current owner scope, keyed by pipeline id and destination id. An absent shared-root owner is publishable for that owner unless a planned output collides with unmanaged storage content. Planned writes to a path owned by another shared-root owner fail as conflicts.
- `publish_new`: no valid state exists and the destination bundle path is empty.
- `upsert_additive`: additive workflow writes planned outputs and retains unrelated catalog outputs.
- `replace_catalog`: replacement workflow writes planned outputs and deletes omitted outputs for the current owner.
- `skip_same`: valid catalog metadata already matches every planned output.
- `force_replace`: explicit bounded replacement selected by `Force`.
- `fail_unmanaged`: unmanaged destination content blocks publication.
- `fail_conflict`: invalid or unsupported state blocks publication.
Execution writes destination state after selected outputs are written. Destination state includes copied source output metadata, generated output metadata, output timestamps, embedded source manifest, reconciliation metadata, link metadata when configured, pipeline id, destination id, and publication timestamps.
Superseded legacy state is identified by schema number and planned through catalog output projection. A successful publish writes schema version `4` catalog state.
## Skip And Resume Behavior
Execution writes destination state after selected outputs are written. Catalog output records include owner identity, compact source identity, copied source output metadata, generated output metadata, output timestamps, and optional URL metadata.
`skip_same` and `skip_destination_newer` execute as no-ops. Replacement-mode single-owner updates remove managed output paths from existing state plus `.distributor.json`, verify the destination is empty, and write state whose outputs are exactly the new plan. Replacement-mode shared-root updates remove only current-owner omitted outputs and preserve unrelated owners. Merge-mode updates retain omitted managed outputs, overwrite only paths already recorded as managed, reject unmanaged destination path collisions, and write cumulative output state. Failed writes trigger cleanup where practical; merge cleanup removes only newly created outputs from the failed attempt.
## Workflow Behavior
Shared-root execution writes schema version `3` state. It preserves unrelated owner records and outputs, updates only the publishing owner metadata, preserves root `created_at`, and updates root `updated_at` after successful state writes. Compatible single-owner state for the same pipeline and destination is converted to shared-root state on successful publish.
Additive workflow computes a write set for the planned outputs and preserves catalog outputs for unplanned paths. Existing catalog records for planned paths are replaced by the current owner and source identity. When every planned output already matches catalog metadata, planning returns `skip_same` and ignores unrelated retained catalog outputs for the no-op decision.
Forced replacement is explicit per request and deletes the bounded destination bundle path before writing new outputs and state.
Replacement workflow computes a write set for the planned outputs, preserves other-owner outputs, and deletes omitted outputs owned by the current pipeline and destination. It does not need `Force`. Matching planned outputs return `skip_same` only when replacement would not delete omitted outputs for the current owner.
Catalog skip comparison is metadata-only. It checks pipeline id, destination id, source id, source digest, source creation timestamp, output path, kind, digest, size, generated output source path, generated output transform, and output URL metadata. It does not read destination file bytes, and `skip_same` execution does not write outputs, rewrite `.distributor.json`, delete files, or notify.
Forced replacement is explicit per request. It deletes the bounded destination bundle path before writing planned outputs and schema version `4` catalog state. Catalog planning selects `force_replace` only when `Force` is true and normal planning would otherwise fail for a non-empty no-state destination, a planned path collision with unmanaged storage content, invalid destination state, or unsupported future destination state.
Retention pruning is not part of publish execution and does not run automatically after a successful publish. The app-level prune workflow uses destination state after publication to select managed outputs for deletion.
## Failure Behavior
Planning fails for incomplete requests, invalid publish/transform policy, invalid state mode, invalid reconciliation mode, output path collisions, invalid destination state, unmanaged destination content without force, shared-root owner path conflicts, conflict outcomes not allowed by transfer policy, unresolved transforms, invalid Markdown output selection, and invalid link URL planning.
Planning fails for incomplete requests, invalid publish/transform policy, invalid workflow, output path collisions, unresolved transforms, invalid Markdown output selection, invalid link URL planning, invalid destination state without force, unmanaged destination content without force, and unsupported future state without force.
Execution fails on delete, read, transform output, unmanaged merge path collision, shared-root ownership conflict, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable publish or replacement actions.
Execution fails on delete, read, transform output, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable catalog publish or replacement actions.
## Tests To Inspect
@@ -62,12 +72,9 @@ Execution fails on delete, read, transform output, unmanaged merge path collisio
- Planning is deterministic for the same request and destination state.
- Destination bundle paths are caller-supplied and backend-root-relative.
- URL generation uses URL path semantics and never infers public URLs from backend config.
- Replacement reconciliation deletes only managed paths recorded in existing state plus `.distributor.json` for single-owner state, and only current-owner omitted outputs for shared-root state.
- Merge reconciliation never adopts unmanaged content.
- Merge state output records are cumulative for the single owner.
- Shared-root planning is owner-scoped and preserves unrelated owner outputs.
- Shared-root execution writes owner-scoped changes without deleting unrelated owners.
- Forced replacement deletes only within the supplied destination bundle path.
- Additive workflow never adopts unmanaged storage content.
- Replacement workflow deletes only catalog outputs owned by the current pipeline and destination unless force replacement clears the bounded destination bundle path.
- Forced replacement deletes only within the supplied destination bundle path and then writes catalog state using the same output projection as normal catalog publish planning.
- Destination state is written after selected outputs are written.
- Transform resolution stays behind a caller-supplied interface.
- Unmanaged content is claimed only by explicit force.

View File

@@ -4,21 +4,21 @@ Audience: developers and LLM coding agents changing `internal/state`.
## Purpose
`internal/state` parses, validates, serializes, and compares `.distributor.json` destination state.
`internal/state` parses, validates, serializes, and updates `.distributor.json` destination catalog state records.
## Inputs And Outputs
Inputs are destination state JSON, constructed state values, current source manifest, pipeline id, destination id, and whether the destination path has content without state. Outputs are validated state values, JSON bytes, comparison outcomes, and human-readable reasons.
Inputs are destination state JSON, constructed catalog values, owner scopes, managed output paths, timestamps, and prune policy inputs. Outputs are validated catalog values, JSON bytes, managed path lists, owner-filtered output lists, missing-output repair results, and prune candidate plans.
## Boundaries
The package does not inspect storage backends, mutate files, choose transfer policy, build publish outputs, generate URLs, or parse config. Publish planning consumes state comparison outcomes.
The package does not inspect storage backends, mutate files, choose workflow actions, build publish outputs, generate URLs, or parse config. Publish planning consumes parsed catalog state and catalog output helpers.
The external destination state contract is documented in `docs/integrations/destination-state.md`.
## Config Fields Used
`internal/state` uses config state mode and reconciliation mode constants for destination state validation and legacy state normalization. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
`internal/state` uses shared constants for catalog mode, output kinds, slug-like id validation, link validation, storage path validation, and source manifest validation. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
## Adapters Used
@@ -26,42 +26,34 @@ None.
## State And Manifest Behavior
`.distributor.json` schema version is `2` for newly written single-owner state. Required fields are `pipeline_id`, `destination_id`, `published_at`, `created_at`, `updated_at`, `state.mode`, `reconciliation.mode`, `source.manifest`, and `outputs`. `distributor_version` and `links` are optional.
Current `.distributor.json` publish output uses schema version `4` catalog state. Required top-level fields are `schema_version`, `created_at`, `updated_at`, `state.mode`, and `outputs`; `distributor_version` is optional.
Schema version `1` state remains readable. Parsing infers `state.mode: single_owner`, `reconciliation.mode: replace`, top-level `created_at` and `updated_at` from `published_at`, and per-output timestamps from `published_at`.
Each catalog output record requires a clean path, pipeline id, destination id, source identity, `source` or `generated` kind, lowercase SHA-256 digest, non-negative size, and created/updated timestamps. Generated outputs require `source_path` and `transform`; copied source outputs must omit both. Stored URLs are optional and must pass `internal/link` validation.
Schema version `3` is shared-root state. It records `state.mode: shared_root`, shared state timestamps, owner records keyed by pipeline id and destination id, each owner's latest source manifest and reconciliation metadata, optional owner primary links, and output records for every managed path. Shared-root output records include owner ids and compact source identity fields for source id, digest, and creation time.
Embedded source identity records contain source manifest id, digest, and creation timestamp. Full source manifests are not embedded in catalog state.
Shared-root publish conversion is explicit. Compatible single-owner state for the same pipeline and destination can be projected into the current owner scope by publish execution. Unrelated single-owner state remains a conflict.
The package identifies schema versions older than the current catalog schema as superseded legacy state for publish planning. It rejects invalid JSON, malformed catalog state, and unsupported future schema versions.
Embedded source manifests are parsed and validated through `internal/bundle`, which delegates source manifest semantics to `pkg/bundle`. Output records require clean paths, `source` or `generated` kind, valid source paths, lowercase SHA-256 digests, non-negative sizes, created and updated timestamps, and transform ids for generated outputs. Stored URLs must pass `internal/link` validation.
The package provides helpers for finding catalog outputs by path, filtering outputs by owner, listing managed output paths, removing missing output records for one owner or every owner, and building owner-scoped prune candidates.
The package also provides helpers for finding output records by path, projecting planned publish outputs into timestamped state outputs, merging retained and newly planned output records, computing managed output paths from single-owner state, removing missing managed output records from single-owner state, and building prune candidates from managed outputs.
For shared-root state, helpers parse either state shape, identify the current owner scope, return an owner's latest source manifest, list managed paths for one owner or all owners, detect path ownership conflicts, project planned owner outputs, merge one owner's planned outputs while preserving unrelated owners, replace one owner's outputs by removing that owner's omitted outputs, remove missing managed output records for either the current owner or every owner, and build owner-scoped prune candidates.
Shared-root helper projections preserve output `created_at` for existing managed paths and use the current publication time for rewritten `updated_at`. Root-level `created_at` preservation is owned by publish execution.
Publish execution owns catalog output projection and timestamp preservation for rewritten outputs. State helpers only parse, validate, filter, and remove catalog records supplied by callers.
## Skip And Resume Behavior
Comparison is pure. It returns outcomes for absent state, unmanaged content, invalid state, pipeline/destination mismatch, same source manifest, older destination, newer destination, same-created digest conflict, and different source id conflict. It does not decide whether to skip, replace, force, or fail; publish planning maps outcomes to actions.
Catalog parsing and helper transformations are pure. State code does not decide whether to skip, upsert, replace, force, or fail; publish planning maps parsed state and storage observations to actions.
Shared-root comparison is owner-scoped. It compares only the owner keyed by the current pipeline id and destination id, treats an absent owner as absent destination state for that owner, and can compare compatible single-owner state for the current owner without converting unrelated single-owner state.
Missing-output removal helpers remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers.
Missing-output removal helpers are pure state transformations used by app-level state repair. They remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers.
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution and the `prune` command use the missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution uses missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
## Failure Behavior
Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid state mode, invalid reconciliation mode, invalid embedded manifests, duplicate owners, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transforms, invalid URLs, invalid digests, negative sizes, and shared-root outputs whose owner is not registered.
Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid catalog mode, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transform metadata, invalid URLs, invalid digests, and negative sizes.
## Tests To Inspect
- `internal/state/distributor_test.go`
- `internal/state/shared_root_test.go`
- `internal/state/catalog_test.go`
- `internal/state/prune_test.go`
- `internal/state/compare_test.go`
- `internal/app/reconcile_state_test.go`
- `internal/cli/reconcile_state_test.go`
- `internal/publish/*_test.go`
@@ -69,15 +61,13 @@ Parsing rejects invalid JSON, trailing data, missing required fields, invalid ti
## Architectural Invariants
- `.distributor.json` is the destination sentinel and state record.
- Comparison does not mutate storage.
- Embedded source manifests use the source bundle contract.
- Newly written single-owner state uses schema version `2`.
- Schema version `1` state remains readable as replacement-mode single-owner state.
- Schema version `3` shared-root state is parsed and validated without converting unrelated single-owner state.
- Shared-root owner updates preserve unrelated owners and reject planned path collisions with other owners.
- State helpers do not inspect or mutate storage.
- Source identity uses the source bundle contract.
- Newly written publish state uses schema version `4`.
- Superseded legacy schema handling is limited to identifying older state for publish planning.
- Missing-output repair helpers preserve unrelated owner records and outputs.
- Prune planning uses output `updated_at` and preserves unrelated shared-root owners.
- Generated outputs always record a transform id.
- Prune planning uses output `updated_at` and preserves unrelated owners.
- Generated outputs always record a transform id and source path.
- Output records always carry created and updated timestamps after parsing.
- Stored URLs are optional and must be absolute HTTP or HTTPS URLs when present.
- `distributor_version` is diagnostic metadata, not a comparison key.

View File

@@ -56,40 +56,33 @@ Destination path mapping controls where each source bundle is published beneath
Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple bundles have the same timestamp, the source-root-relative bundle path in ascending order wins.
Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination state record. It records the pipeline id, destination id, publication time, state mode, reconciliation mode, source manifest, copied outputs, generated outputs, output timestamps, and optional public URL metadata.
Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination catalog. It records catalog schema version `4`, output owners, source identity for each output, output digests and sizes, timestamps, and optional URL metadata.
`manifest.json` from the source bundle is not copied as destination state.
## Destination State And Retry Behavior
## Catalog Publish Behavior
`distributor` compares the source manifest to destination `.distributor.json` before writing:
`distributor` plans from the current source bundle, destination workflow, destination storage content, and `.distributor.json`.
- No destination state and no destination content: publish new outputs.
- Matching destination state: skip as already published.
- Older destination state for the same source id: replace if transfer policy allows it.
- Newer destination state: skip by default.
- Invalid destination state, identity mismatch, different source id, or same-created digest mismatch: fail by default.
- Content without `.distributor.json`: fail as unmanaged content by default.
- No valid state and no destination content: publish new outputs.
- No valid state and existing destination content: fail as unmanaged unless `--force` is used.
- Valid catalog state with `workflow: additive`: write planned outputs and retain unrelated catalog outputs.
- Valid catalog state with `workflow: replacement`: write planned outputs and remove omitted outputs owned by the selected pipeline and destination.
- Planned output path exists in storage but is not recorded in valid catalog state - fail as unmanaged unless `--force` is used.
- Invalid destination state or unsupported future state - fail as conflict unless `--force` is used.
- Superseded legacy state - publish through the catalog planner and write schema version `4` state on success.
When destination state is older than the source, `transfer.on_destination_older` controls whether publication may proceed and `reconciliation.mode` controls how managed outputs are updated.
`workflow: additive` is the default. It is useful for archive roots, fan-out roots that intentionally receive disjoint outputs, and roots that accumulate managed outputs over time.
`reconciliation.mode: replace` is the default. It deletes only managed output paths recorded in `.distributor.json` plus the state file, verifies the destination bundle path is empty, then writes the newly planned outputs and state. The new state `outputs` array is exactly the newly planned output set.
`reconciliation.mode: merge` retains prior managed outputs that are omitted from the new plan. It overwrites planned paths only when those paths are already recorded in existing state as managed. If a newly planned path already exists in storage but is not recorded in state, publication fails as an unmanaged path collision. The new state `outputs` array is the cumulative managed output set.
For both modes, retained or overwritten paths are identified only from `.distributor.json`; unmanaged files are not adopted.
For `state.mode: shared_root`, one destination root may contain outputs from multiple pipeline/destination owners. Comparisons, replacement, and merge retention are scoped to the current owner. Outputs owned by other owners are preserved. A planned output path owned by another owner fails as a conflict, and a planned path that exists in storage but is not recorded in state fails as unmanaged content by default.
If `state.mode: shared_root` is configured on a destination whose existing single-owner state belongs to the same pipeline and destination, the next successful publish converts that state file to shared-root schema. Existing single-owner state for a different pipeline or destination remains a conflict.
If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. In `replace` mode, cleanup removes outputs written during that failed attempt. In `merge` mode, cleanup removes only newly created outputs from that failed attempt; overwritten managed outputs are left in place because they previously belonged to the managed set. Operators should still inspect the destination after a failed write before retrying.
`workflow: replacement` is useful for stable latest-style roots where the current pipeline and destination should leave only the currently planned output set for that owner. Replacement workflow is normal managed behavior and does not require `--force`.
Fan-out destinations are independent. If one destination fails after planning or execution begins, later destinations are still attempted. The command exits non-zero if any destination failed.
If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. Operators should still inspect the destination bundle path after a failed write before retrying.
## Destination State Repair
Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the state record only; it does not restore missing files.
Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the catalog record only; it does not restore missing files.
Preview the repair first:
@@ -110,11 +103,9 @@ go run ./cmd/distributor reconcile-state \
--destination <destination-id>
```
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks each managed output path with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting.
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks matching catalog output paths with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting.
Without `--dry-run`, it removes missing managed output records from valid state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid or mismatched state.
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, repair is scoped to the selected owner by default. Add `--all-owners` only when every owner in the selected shared-root state should have missing managed output records removed.
Without `--dry-run`, it removes missing managed output records from valid catalog state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid state. Add `--all-owners` only when every catalog owner inside the selected root should be repaired.
## Managed Output Pruning
@@ -142,28 +133,28 @@ go run ./cmd/distributor prune \
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, and plans from the selected destination's `retention.prune` policy. It uses managed output `updated_at` timestamps. When both `keep_latest` and `older_than` are configured, it preserves the newest `keep_latest` outputs before applying the age policy.
`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate.
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths for the selected pipeline/destination owner, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate.
## Dry Runs And Output Review
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, or SSH `known_hosts` entries. For reconciliation, dry runs report the same high-level action labels as execution; inspect the configured destination's `reconciliation.mode` to determine whether `replace_older` will replace the managed set or merge into it.
For shared-root destinations, dry runs are owner-scoped. A `replace_older` action replaces or merges only the current owner according to `reconciliation.mode`; unrelated owners remain managed by the shared-root state.
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, notifier events, or SSH `known_hosts` entries.
Review these action labels before publishing:
- `publish_new`: destination state is absent, or a shared-root owner is absent and planned paths are publishable.
- `replace_older`: destination state is older than the source.
- `skip_same`: destination state already matches the source.
- `skip_destination_newer`: destination state is newer than the source and is skipped.
- `force_replace`: destructive replacement selected because `--force` is present and policy permits it.
- `error`: planning or execution failed for that destination.
- `publish_new`: destination state is absent and the destination bundle path is empty.
- `upsert_additive`: additive workflow will write planned outputs into valid catalog state.
- `replace_catalog`: replacement workflow will write planned outputs and remove omitted outputs for the current owner.
- `skip_same`: planned outputs already match valid catalog metadata, so publication will not write outputs, rewrite `.distributor.json`, delete files, or notify.
- `force_replace`: destructive catalog replacement selected because `--force` is present for unmanaged content, a planned unmanaged path collision, invalid state, or unsupported future state.
- `fail_unmanaged`: unmanaged destination content prevents publication.
- `fail_conflict`: invalid or unsupported destination state prevents publication.
- `error`: setup, planning, or execution failed for that destination.
Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced.
Text and JSON summaries count `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, `fail_conflict`, and failed destinations separately. JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document.
JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document.
The `skip_same` optimization trusts valid catalog metadata. It compares owner identity, source identity, output path, kind, digest, size, generated output metadata, and URL metadata recorded in `.distributor.json`; it does not read destination file bytes to detect bitrot.
Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced. For fixed destinations, the resolved destination bundle path is the backend root.
## Forced Replacement Workflow
@@ -174,14 +165,9 @@ go run ./cmd/distributor run --config <config-path> --dry-run --force
go run ./cmd/distributor run --config <config-path> --force
```
Forced replacement can claim unmanaged non-empty destination paths. State conflicts require both `--force` and transfer policy that permits replacement:
Forced replacement can claim a non-empty destination path with no valid `.distributor.json`, replace planned output paths that collide with storage content not recorded in valid catalog state, and recover from invalid or unsupported future destination state. It is reserved for exceptional destructive replacement. Valid catalog-managed additive upserts and replacement workflow publishes do not require `--force`.
- newer destination state requires `transfer.on_destination_newer: replace`;
- conflict outcomes require `transfer.on_conflict: replace`.
Forced replacement deletes the current destination bundle path before writing outputs and state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the backend root, so a forced replacement can clear that configured root.
For shared-root destinations, forced replacement also deletes the configured destination bundle path before writing new shared-root state. This removes unrelated owners inside that destination root. Preview with `--dry-run --force` and confirm the destination path before applying.
Forced replacement deletes the current destination bundle path before writing planned outputs and schema version `4` catalog state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the configured backend root, so a forced replacement can clear that configured root. Dry-run text and JSON output report that root as `target=.` or `destination_path: "."`.
`--force` applies only to the current invocation. There is no config field that enables forced replacement by default.
@@ -221,9 +207,9 @@ curl -X POST http://127.0.0.1:8080/v1/pipelines/example-http-upload/upload \
--data-binary @bundle.tar.gz
```
Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md) for the copyable producer implementation guide.
Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md).
The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`. Set `DISTRIBUTOR_EXAMPLE_UPLOAD_PIPELINE_ID` or pass a second argument to use another configured upload pipeline. It generates an idempotency key by default; set `DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY` when retrying the same producer run across separate process runs.
The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`.
```sh
go run ./examples/upload-client
@@ -245,9 +231,9 @@ Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Co
Upload admission is bounded by `server.http.queue_size`. Publication concurrency is bounded by `server.http.max_concurrency`, and the coordinator does not run two uploads for the same pipeline at the same time.
`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`. If another request with the same key is still being staged before its manifest is known, the server returns a retryable `409 Conflict`. Idempotency records are memory-only and expire with completed upload status records.
`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`.
The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size. The implementation also caps extracted file count.
The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size.
The default bind address is private loopback. Put TLS, public routing, rate limiting, and external access policy in a reverse proxy or deployment layer.
@@ -267,7 +253,7 @@ S3 execution uses the AWS SDK for Go v2. See [S3-Compatible Storage Integration]
When explicit S3 credential variable names are configured, both must resolve to non-empty values through the process environment or `secrets.directory`. When omitted, the AWS SDK default credential chain is used as-is.
Normal single-owner replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Shared-root replacement deletes only current-owner omitted output objects and rewrites the shared state object. Merge publication retains omitted managed objects and may overwrite existing managed objects. Forced replacement deletes objects under the bounded destination bundle prefix. Distributor does not manage bucket versioning or delete markers.
Normal managed replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Forced replacement deletes objects under the bounded destination bundle prefix and then writes schema version `4` catalog state. Distributor does not manage bucket versioning or delete markers.
## Secrets Operation
@@ -282,11 +268,10 @@ Use these recovery boundaries:
- For source validation failures, regenerate the source bundle and manifest together.
- For an empty or missing destination, rerun after fixing config or storage access.
- For unmanaged destination content, move unrelated files aside or use a different destination path before publishing.
- For shared-root ownership conflicts, change one owner so it writes a different destination path, or use a separate destination root.
- For invalid or unsupported destination state, inspect `.distributor.json`; use `--force` only after dry-run review confirms bounded replacement is intended.
- For missing managed output files recorded in state, run `reconcile-state --dry-run`, then apply `reconcile-state` if the missing files should no longer be considered managed.
- For configured retention cleanup, run `prune --dry-run`, then apply `prune --apply` after reviewing the managed output list.
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`. In merge mode, retained outputs may be intentional managed outputs from the prior state.
- For state conflicts, verify the source, pipeline, destination, and existing `.distributor.json` before considering `--force`.
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`.
- For HTTP upload failures, inspect `/runs/<run-id>` while retained; after expiry or restart, rely on destination state and logs/output from the publishing run.
Do not edit `.distributor.json` during normal recovery. Treat it as the managed state record used for comparison and safe cleanup.

View File

@@ -96,18 +96,15 @@ The source manifest should remain minimal. Routing, destination selection, publi
Each destination bundle path is managed by `.distributor.json`. This file is both the destination sentinel and the destination state record.
Publish execution writes single-owner or shared-root destination state according to destination `state.mode`. In shared-root state, one `.distributor.json` records multiple pipeline/destination owners and every managed output carries its owner identity.
Publish execution writes catalog destination state. One `.distributor.json` records all managed outputs under the destination bundle path, and each output carries its owning pipeline id and destination id.
Single-owner state records:
Catalog state records:
- `distributor` state schema version;
- pipeline id;
- destination id;
- publication timestamp;
- state creation and update timestamps;
- single-owner state mode;
- reconciliation mode;
- the normalized source manifest used for publication;
- catalog state mode;
- owner identity for each managed output;
- compact source identity for each managed output;
- metadata for copied source outputs;
- metadata for generated outputs, such as HTML files;
- optional URL metadata for published outputs;
@@ -117,40 +114,23 @@ A representative destination state file is:
```json
{
"schema_version": 2,
"schema_version": 4,
"distributor_version": "0.1.0",
"pipeline_id": "weather-daily",
"destination_id": "static-html",
"published_at": "2026-05-30T11:12:00Z",
"created_at": "2026-05-30T11:12:00Z",
"updated_at": "2026-05-30T11:12:00Z",
"state": {
"mode": "single_owner"
},
"reconciliation": {
"mode": "replace"
},
"source": {
"manifest": {
"schema_version": 1,
"id": "weather.daily.brentwood.2026-05-30",
"digest": "sha256:...",
"created": "2026-05-30T11:10:00Z",
"files": [
{
"path": "report.md",
"sha256": "sha256:...",
"size": 12345
}
]
}
},
"links": {
"primary_url": "https://reports.example.com/weather-daily/"
"mode": "catalog"
},
"outputs": [
{
"path": "index.html",
"pipeline_id": "weather-daily",
"destination_id": "static-html",
"source": {
"id": "weather.daily.brentwood.2026-05-30",
"digest": "sha256:...",
"created": "2026-05-30T11:10:00Z"
},
"kind": "generated",
"source_path": "report.md",
"transform": "markdown_to_html",
@@ -167,13 +147,11 @@ A representative destination state file is:
Destination comparison rules are based on `.distributor.json`:
- No `.distributor.json`: publish normally only if the destination bundle path is empty.
- Existing state embeds the same normalized source manifest: skip as already published.
- Existing state has the same source id and an older source `created`: replace, subject to destructive-operation safety rules.
- Existing state has the same source id and a newer source `created`: skip because the destination is newer than the source.
- Existing state has the same source id and same `created` but different digest: fail as a conflict.
- Existing state has a different source id: fail as a conflict.
For older destination state, destination `reconciliation.mode` controls output updates. `replace` rewrites the managed output set to match the new plan. `merge` retains omitted managed outputs, overwrites only existing managed paths, and rejects planned paths that collide with unmanaged storage content.
- Existing catalog state with additive workflow: write planned outputs and retain unrelated managed outputs.
- Existing catalog state with replacement workflow: write planned outputs and remove omitted outputs for the current pipeline and destination owner.
- Planned paths that collide with unmanaged storage content fail by default.
- Invalid or unsupported destination state fails by default.
- Explicit forced replacement may clear the bounded destination bundle path after dry-run review.
## Publication and Transform Policy
@@ -200,7 +178,7 @@ Application logic must interact with storage through internal backend interfaces
Adapters should be thin. Backend adapters should implement storage operations and translate backend-specific errors, but should not make bundle comparison, transform, routing, or replacement decisions.
Remote file-transfer support should prefer native protocol implementations over shelling out, unless a later design document records a reason to differ.
Remote file copy support should prefer native protocol implementations over shelling out, unless a later design document records a reason to differ.
## Dependency Policy
@@ -221,7 +199,7 @@ Use this current layout unless the project has a documented reason to differ:
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
- `internal/bundle`: storage-backed source bundle discovery and validation over the public manifest contract.
- `internal/state`: `.distributor.json` parsing, validation, comparison, and output metadata.
- `internal/state`: `.distributor.json` catalog parsing, validation, and output metadata.
- `internal/link`: shared HTTP URL validation for configured and persisted link metadata.
- `internal/storage`: backend interfaces, shared path/resource types, backend registry, and storage errors.
- `internal/adapters/local`: local filesystem backend.
@@ -229,7 +207,7 @@ Use this current layout unless the project has a documented reason to differ:
- `internal/adapters/s3`: S3-compatible object storage backend.
- `internal/transform`: transform interfaces, registry, planning, and shared transform models.
- `internal/transform/markdown`: Markdown-to-HTML implementation.
- `internal/publish`: destination planning, reconciliation, safety checks, and publish execution.
- `internal/publish`: destination planning, catalog workflow safety checks, and publish execution.
- `internal/notify`: notification interface and MVP no-op notifier.
- `internal/logging`: logging setup and shared logging helpers.
@@ -264,7 +242,7 @@ Pipeline configuration should express:
- per-destination transform policy;
- per-destination public link policy;
- validation behavior;
- destination conflict/replacement behavior.
- per-destination workflow and retention behavior.
## Modules and Registries
@@ -294,7 +272,7 @@ Errors should be actionable and preserve context. Wrap errors with operation, pi
Errors and logs must not expose secrets.
Use structured logging where practical. Logs should describe discovery, validation, planned actions, skipped transfers, conflicts, replacements, external calls, retries, and failure causes, but should not include large report contents by default.
Use structured logging where practical. Logs should describe discovery, validation, planned actions, skipped copies, conflicts, replacements, external calls, retries, and failure causes, but should not include large report contents by default.
Skip and no-op decisions should be logged at an appropriate level so operators can distinguish successful publication from intentional no-op behavior.

View File

@@ -12,13 +12,13 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
- `internal/cli`: standard-library command parsing, flags, help text, and command wiring.
- `internal/config`: YAML configuration structs, loading, defaults, and validation.
- `internal/bundle`: storage-backed source bundle discovery and validation using the public manifest contract.
- `internal/state`: destination `.distributor.json` parsing, validation, and comparison.
- `internal/state`: destination `.distributor.json` catalog parsing, validation, and output metadata.
- `internal/storage`: backend interface, registry, logical path rules, typed errors, and shared storage helpers.
- `internal/adapters/local`: local filesystem backend.
- `internal/adapters/ssh`: SSH/SFTP backend.
- `internal/adapters/s3`: S3-compatible object storage backend.
- `internal/storage/fake`: in-memory backend for tests.
- `internal/publish`: destination inspection, output planning, reconciliation, execution, managed cleanup, and explicit forced replacement.
- `internal/publish`: destination inspection, output planning, catalog workflow planning, execution, managed cleanup, and explicit forced replacement.
- `internal/transform`: transform interface and registry.
- `internal/transform/markdown`: Markdown-to-HTML transform.
- `internal/notify`: notification interface and current no-op notifier.

View File

@@ -43,6 +43,7 @@ Canonical homes:
- project purpose and quickstart: `README.md`
- development principles: `docs/policy/architecture.md`
- public HTTP API reference: `docs/api.md`
- configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md`
@@ -122,6 +123,22 @@ Recommended:
- `docs/troubleshooting.md`
- validated examples under `examples/`
### Public HTTP API service
Required:
- `docs/api.md`
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- `docs/consumers/`, for task-oriented client integration guides
- `docs/integrations/`, for upstream/downstream service contracts
- validated examples under `examples/`
### Project with public packages or consumer APIs
Required:
@@ -173,6 +190,32 @@ It should include:
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
### docs/api.md
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
Required for projects whose primary public interface is HTTP.
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
It should include:
1. base URL conventions;
2. authentication and authorization behavior, if implemented;
3. response envelope;
4. supported media types and content negotiation behavior;
5. shared query parameters;
6. endpoint reference grouped by route family;
7. request parameters and validation rules;
8. response fields, units, nullability, and optionality;
9. error response shape and status codes;
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
11. compact request and response examples.
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
### docs/policy/development.md
**Audience:** developers, LLM coding agents
@@ -264,6 +307,8 @@ Required for projects with public packages, SDKs, client APIs, plugin APIs, or o
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
1. intended consumer audience and use cases;
@@ -330,6 +375,8 @@ Required for projects that depend on external CLIs, APIs, services, protocols, o
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
Use one file per integration where useful.
## Examples Directory
@@ -385,9 +432,10 @@ Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/policy/architecture.md` describes development principles.
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Consumer-facing docs explain public APIs without duplicating integration contracts.
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.

View File

@@ -93,10 +93,10 @@ internal docs.
- Current upload status, queue, and idempotency state are memory-only.
- Producers submit complete tar or gzip-compressed tar source bundles today.
- Producers do not choose destination ids, destination paths, transforms, links,
publish policy, transfer policy, storage backends, reconciliation mode, state
mode, or retention policy through upload requests.
publish policy, destination workflow, storage backends, or retention policy
through upload requests.
- Source manifests remain free of routing, destination, transform, credential,
reconciliation, state, and retention data.
workflow, state, and retention data.
- Public access policy, TLS termination, and rate limiting belong outside
`distributor` unless a future implementation changes that boundary.
- `distributor` is not a broad storage synchronization tool unless a future

View File

@@ -24,7 +24,9 @@ Reference: [Configuration](config.md#config-file-loading).
Symptom: `parse config ... field not found`.
Likely cause: the YAML contains a key that is not part of the implemented config schema.
Likely cause: the YAML contains a key that is not part of the implemented
config schema. Pre-workflow destination policy keys for state mode, conflict
handling, ownership adoption, or per-comparison copy decisions are rejected.
Diagnostic:
@@ -32,9 +34,9 @@ Diagnostic:
go run ./cmd/distributor run --config <config-path> --dry-run
```
Safe fix: remove or rename unsupported fields using the canonical config reference.
Safe fix: remove unsupported fields using the canonical config reference. Destination behavior is configured with `workflow`, `publish`, `transform`, `path_mapping`, `links`, and `retention`.
Reference: [Configuration](config.md).
Reference: [Configuration](config.md#destination-fields).
## Backend Name Or Placement Is Invalid
@@ -71,9 +73,9 @@ Reference: [CLI](cli.md#validate).
## Reconcile-State Selector Is Missing Or Wrong
Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "<id>" not found`, or `destination <id> not found`.
Likely cause: the command did not identify one configured destination root, or the selected root contains state for a different single-owner pipeline/destination.
Likely cause: the command did not identify one configured destination root.
Diagnostic:
@@ -83,15 +85,15 @@ rg -n 'pipelines:|destinations:|id:' <config-path>
cat <destination-path>/.distributor.json
```
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. For unrelated single-owner state, use the correct config selector or a separate destination root; `reconcile-state` will not take over mismatched state.
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file.
Reference: [CLI](cli.md#reconcile-state).
## Prune Selector Or Mode Is Missing Or Wrong
Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "<id>" not found`, or `destination <id> not found`.
Likely cause: the command did not identify one configured destination root, did not choose exactly one execution mode, or the selected root contains state for a different single-owner pipeline/destination.
Likely cause: the command did not identify one configured destination root or did not choose exactly one execution mode.
Diagnostic:
@@ -101,7 +103,7 @@ rg -n 'retention:|prune:|pipelines:|destinations:|id:' <config-path>
cat <destination-path>/.distributor.json
```
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both. For unrelated single-owner state, use the correct config selector or a separate destination root; `prune` will not take over mismatched state.
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both.
Reference: [CLI](cli.md#prune).
@@ -187,9 +189,9 @@ Reference: [Operations](operations.md#cleanup-and-recovery).
## Destination Has Unmanaged Content
Symptom: `destination has content but no distributor state` or a plan reason containing `fail_unmanaged`.
Symptom: `destination has content but no distributor state`, `destination output path ... exists but is not managed by catalog state`, or a plan reason containing `fail_unmanaged`.
Likely cause: the destination bundle path contains files but no valid `.distributor.json`, so `distributor` will not claim it by default.
Likely cause: the destination bundle path contains files but no valid `.distributor.json`, or a planned output path collides with storage content that valid catalog state does not record. `distributor` will not claim unmanaged content by default.
Diagnostic:
@@ -197,26 +199,43 @@ Diagnostic:
find <destination-path> -maxdepth 2 -print
```
Safe fix: choose an empty destination path, move unrelated files aside, or preview `run --dry-run --force` only after confirming the reported bundle path is safe to replace.
Safe fix: choose an empty destination path, move unrelated files aside, or preview `run --dry-run --force` only after confirming the reported destination bundle path is safe to replace.
Reference: [Operations](operations.md#forced-replacement-workflow).
## Destination State Conflicts With Source
## Destination State Is Invalid Or Unsupported
Symptom: `fail_conflict`, `destination source id differs`, `same id and created time but different digest`, `pipeline id ... does not match`, or `destination id ... does not match`.
Symptom: `fail_conflict`, `parse distributor state`, `state schema_version must be 4`, or `unsupported future destination state`.
Likely cause: `.distributor.json` belongs to a different pipeline, destination, source id, or same-created source with different content.
Likely cause: `.distributor.json` is invalid JSON, has invalid catalog fields, or uses an unsupported future schema.
Diagnostic:
```sh
cat <destination-path>/.distributor.json
go run ./cmd/distributor inspect <source-root>
go run ./cmd/distributor run --config <config-path> --dry-run --format json
```
Safe fix: verify the source and destination are intended to match. Use a separate destination path for unrelated content. To replace the existing state, configure `transfer.on_conflict: replace`, preview with `--dry-run --force`, then publish with `--force`.
Safe fix: restore a valid catalog state file from backup, choose a different destination path, or use `--force` only after `run --dry-run --force` reports the intended bounded `force_replace`.
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
Reference: [Operations](operations.md#forced-replacement-workflow).
## Destination Uses Superseded Legacy State
Symptom: dry-run reports a normal catalog action against an older `.distributor.json`, or `reconcile-state` / `prune` reports that the destination state schema is superseded.
Likely cause: the destination contains a state file written by an older implementation. Publish planning can replace it with catalog state on successful `run`, but maintenance commands only operate on current catalog state.
Diagnostic:
```sh
cat <destination-path>/.distributor.json
go run ./cmd/distributor run --config <config-path> --dry-run
```
Safe fix: preview the publish plan, then run publication if the destination path is correct. The successful run writes schema version `4` catalog state.
Reference: [Destination State Contract](integrations/destination-state.md).
## Destination State References Missing Managed Outputs
@@ -242,7 +261,7 @@ Reference: [Operations](operations.md#destination-state-repair).
Symptom: `prune --dry-run` reports `planned=0` or JSON `planned_outputs: []`.
Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected shared-root owner has no eligible outputs.
Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected pipeline/destination owner has no eligible outputs.
Diagnostic:
@@ -260,27 +279,11 @@ Safe fix: verify the selected destination's retention policy and the managed out
Reference: [Operations](operations.md#managed-output-pruning).
## Destination Is Newer Than Source
Symptom: `skip_destination_newer` or `destination is newer and replacement requires --force`.
Likely cause: the destination state records a source manifest with a later `created` timestamp than the current source.
Diagnostic:
```sh
go run ./cmd/distributor run --config <config-path> --dry-run --format json
```
Safe fix: keep the default skip behavior unless replacement is intentional. To replace newer state, configure `transfer.on_destination_newer: replace`, preview with `--dry-run --force`, then publish with `--force`.
Reference: [Operations](operations.md#forced-replacement-workflow).
## Forced Replacement Appears In A Plan
Symptom: dry-run output includes `force_replace`.
Likely cause: the run used `--force`, and planning selected a supported destructive replacement.
Likely cause: the run used `--force`, and catalog planning selected a supported destructive replacement for unmanaged destination content, a planned unmanaged path collision, invalid destination state, or unsupported future destination state.
Diagnostic:
@@ -288,15 +291,15 @@ Diagnostic:
go run ./cmd/distributor run --config <config-path> --dry-run --force
```
Safe fix: inspect the pipeline id, destination id, backend, and bundle path. Proceed only if deleting everything inside that destination bundle path is intended.
Safe fix: inspect the pipeline id, destination id, backend, and bundle path. Confirm the `destination_path` in JSON output, or the fixed-path `target=.` text output, before applying. Proceed only if deleting everything inside that destination bundle path is intended; `force_replace` then writes planned outputs and schema version `4` catalog state.
Reference: [Operations](operations.md#forced-replacement-workflow).
## Output Path Collision
Symptom: `destination output path collision` or `merge output path ... exists but is not managed by destination state`.
Symptom: `destination output path collision` or `destination output path ... exists but is not managed by catalog state`.
Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`. For merge reconciliation, it can also mean a planned output path already exists in storage but is not recorded in `.distributor.json` as managed.
Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`, or a planned output path already exists in storage but is not catalog-managed.
Diagnostic:
@@ -304,44 +307,10 @@ Diagnostic:
go run ./cmd/distributor run --config <config-path> --dry-run
```
Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For merge reconciliation, move unmanaged content aside, choose another destination path, or use replacement/forced replacement only when deleting the destination bundle path is intended.
Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For unmanaged storage collisions, move the unmanaged file aside, choose another destination path, or use forced replacement only when deleting the destination bundle path is intended.
Reference: [Configuration](config.md#publish-and-transform-policy).
## Shared-Root Ownership Conflict
Symptom: `fail_conflict` with a reason like `destination output path ... is owned by <pipeline>/<destination>`.
Likely cause: a `state.mode: shared_root` destination planned an output path already recorded in `.distributor.json` for another pipeline/destination owner.
Diagnostic:
```sh
cat <destination-path>/.distributor.json
go run ./cmd/distributor run --config <config-path> --dry-run
```
Safe fix: change one owner so it writes a different output path, use a separate destination root, or intentionally replace the whole destination root only after previewing with `--dry-run --force`.
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
## Shared-Root Unmanaged Path Collision
Symptom: `fail_unmanaged` with a reason like `destination output path ... exists but is not managed by destination state`.
Likely cause: a `state.mode: shared_root` destination planned a new output path that already exists in storage but is not recorded as managed in `.distributor.json`.
Diagnostic:
```sh
find <destination-path> -maxdepth 2 -print
cat <destination-path>/.distributor.json
```
Safe fix: move the unmanaged file aside, change the planned output path, or use forced replacement only when deleting the configured destination root is intended.
Reference: [Operations](operations.md#forced-replacement-workflow).
## Run Failed After Writing Some Files
Symptom: a destination write failed and the command exited non-zero after partial work.
@@ -354,9 +323,9 @@ Diagnostic:
find <destination-path> -maxdepth 2 -print
```
Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. In merge mode, previously managed retained or overwritten outputs may remain intentionally. Rerun `--dry-run` before publishing again.
Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. Rerun `--dry-run` before publishing again.
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
Reference: [Operations](operations.md#catalog-publish-behavior).
## JSON Run Result Has `ok: false`
@@ -378,240 +347,14 @@ Reference: [CLI](cli.md#output-and-exit-behavior).
Symptom: `load secrets directory ... no such file or directory`, `permission denied`, or `secret filename ... is invalid`.
Likely cause: `secrets.directory` points to a missing or unreadable directory, or it contains a regular file whose name is not a valid credential variable name.
Likely cause: `secrets.directory` points to a missing or unreadable directory, or contains a filename that cannot be used as a credential variable name.
Diagnostic:
```sh
ls -ld <secrets-directory>
find <secrets-directory> -maxdepth 1 -type f -printf '%f\n'
ls -la <secrets-directory>
```
Safe fix: mount or create the directory, adjust permissions for the service user, or rename/remove invalid secret files. Secret filenames must match `[A-Za-z_][A-Za-z0-9_]*`.
Safe fix: create the directory, fix permissions, or rename secret files to valid environment-variable-style names.
Reference: [Configuration](config.md#secrets).
## Credential Variable Is Missing Or Empty
Symptom: `credential environment variable ... is not set`, `credential environment variable ... is empty`, or S3 authentication errors such as `AccessDenied`, `InvalidAccessKeyId`, or `SignatureDoesNotMatch`.
Likely cause: configured S3 credential variable names are not available through the process environment or `secrets.directory`, are empty, or do not authorize the requested bucket/prefix.
Diagnostic:
```sh
env | cut -d= -f1 | rg '^(<access-key-variable>|<secret-key-variable>)$'
ls -l <secrets-directory>
```
Safe fix: provide both configured S3 credential values, correct IAM/service permissions, or omit explicit credential fields to use the AWS SDK default credential chain.
Reference: [Configuration](config.md#s3-compatible-backend).
## Secret File Is Ignored In Favor Of Environment
Symptom: `secret ... ignored because the real environment already has that variable`.
Likely cause: the same credential name exists in the process environment and `secrets.directory` with different values.
Diagnostic:
```sh
env | cut -d= -f1 | rg '^<variable-name>$'
ls -l <secrets-directory>/<variable-name>
```
Safe fix: remove one source of the credential or intentionally keep the process environment value. `distributor` does not print either value.
Reference: [Operations](operations.md#secrets-operation).
## SSH Auth Is Not Configured
Symptom: `no SSH auth methods configured`.
Likely cause: no SSH agent is available and `ssh_key_file` is missing or unreadable.
Diagnostic:
```sh
test -n "$SSH_AUTH_SOCK" && ssh-add -l
ls -l <ssh-key-file>
```
Safe fix: start an SSH agent with an appropriate key loaded, or configure a readable private key with `ssh_key_file`.
Reference: [Configuration](config.md#sshsftp-backend).
## SSH Host Key Fails
Symptom: `host key ... is unknown`, `known_hosts is required`, or `host key ... has changed`.
Likely cause: strict host key checking has no trusted key, `accept-new` cannot persist a new key, or the remote host key differs from the stored key.
Diagnostic:
```sh
ls -l <known-hosts-path>
ssh-keygen -F <host> -f <known-hosts-path>
```
Safe fix: pre-populate `known_hosts` for `strict`, configure a writable `known_hosts` path for `accept-new`, or verify the server identity before updating a changed key. Do not disable host key checking to bypass an unexpected changed key.
Reference: [Operations](operations.md#sshsftp).
## S3 Prefix Is Invalid
Symptom: `prefix must be a clean relative slash-separated path`.
Likely cause: the S3 prefix contains traversal, dot segments, empty segments, or backslashes after leading and trailing slashes are trimmed.
Diagnostic:
```sh
go run ./cmd/distributor run --config <config-path> --dry-run
```
Safe fix: use a clean relative prefix such as `reports/archive`, or omit `prefix`.
Reference: [Configuration](config.md#s3-compatible-backend).
## S3 Location Or Connectivity Fails
Symptom: `NoSuchBucket`, `InvalidBucketName`, `not_found`, endpoint connection failures, or TLS/network errors.
Likely cause: endpoint, bucket, prefix, region, path-style mode, network routing, or credentials are wrong for the service.
Diagnostic:
```sh
go run ./cmd/distributor run --config <config-path> --dry-run
curl -I <endpoint>
```
Safe fix: verify `endpoint`, `bucket`, `region`, `prefix`, and `force_path_style`. For S3-compatible services, keep `force_path_style: true` unless the service requires virtual-host addressing. Distributor does not provide an insecure TLS bypass setting.
Reference: [Operations](operations.md#s3-compatible-storage).
## HTTP Server Cannot Bind
Symptom: `bind HTTP server ... address already in use`.
Likely cause: another process is listening on `server.http.bind`.
Diagnostic:
```sh
ss -ltnp | rg '<port>'
```
Safe fix: stop the conflicting process or configure a different bind address.
Reference: [Configuration](config.md#serverhttp).
## HTTP Upload Token Is Missing Or Duplicated
Symptom: `upload token environment variable ... is not set`, `... is empty`, or `upload token environment variables ... resolve to the same value`.
Likely cause: a top-level upload token record references a missing or empty `token_env`, or two token records resolve to the same bearer token.
Diagnostic:
```sh
rg -n 'token_env:' <config-path>
env | cut -d= -f1 | rg '^<token-variable>$'
ls -l <secrets-directory>/<token-variable>
```
Safe fix: provide one distinct non-empty token value per upload token record through the process environment or `secrets.directory`. Do not put literal tokens in YAML.
Reference: [Configuration](config.md#upload_tokens).
## Upload Request Is Unauthorized
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `401`.
Likely cause: the request lacks `Authorization: Bearer <token>`, has an empty token, or uses a token that does not match any configured upload token record.
Diagnostic:
```sh
curl -i -X POST http://127.0.0.1:8080/v1/pipelines/<pipeline-id>/upload \
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
-H "Content-Type: application/x-tar" \
--data-binary @bundle.tar
```
Safe fix: use the token value resolved by the configured `token_env`. Do not include token values in logs or tickets.
Reference: [Operations](operations.md#http-upload-operation).
## Upload Request Is Forbidden
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `403`.
Likely cause: the bearer token is valid, but its configured `allow_pipelines` list does not include the requested upload pipeline.
Diagnostic:
```sh
rg -n 'upload_tokens:|allow_pipelines:|id:' <config-path>
```
Safe fix: request the intended pipeline id, or update the token allowlist to include the configured `http_upload` pipeline that this producer may submit to.
Reference: [Configuration](config.md#upload_tokens).
## Upload Request Is Rejected Before A Run ID
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `400`, `413`, `415`, or `503`.
Likely cause: the request path has an invalid pipeline id, included a `pipeline` or `pipeline_id` query, archive content is malformed, the body exceeds size limits, content type is unsupported, or the in-memory upload queue is full.
Diagnostic:
```sh
tar -tf bundle.tar
tar -tzf bundle.tar.gz
rg -n 'max_upload_size|queue_size|max_concurrency' <config-path>
```
Safe fix: send one valid tar or tar.gz source bundle archive to `/v1/pipelines/<pipeline-id>/upload` with `Content-Type: application/x-tar`, `application/gzip`, or `application/x-gzip`; remove pipeline query parameters; reduce archive size or raise the configured limit; retry after queue pressure drops.
Reference: [Operations](operations.md#http-upload-operation).
## Upload Idempotency Conflict
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `409`.
Likely cause: the request reused an `Idempotency-Key` for the same token id and pipeline id with a different source manifest, or another request with the same key is still being staged before its manifest is known.
Diagnostic:
```sh
curl -i -X POST http://127.0.0.1:8080/v1/pipelines/<pipeline-id>/upload \
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
-H "Content-Type: application/gzip" \
-H "Idempotency-Key: <key>" \
--data-binary @bundle.tar.gz
```
Safe fix: if the response includes `"retryable":true`, retry the same upload later with the same key. Otherwise, inspect the producer operation and use the same key only for the same source bundle.
Reference: [HTTP Upload API Contract](integrations/http-upload.md).
## Upload Status Is Missing
Symptom: `GET /runs/<run_id>` returns `404`.
Likely cause: the run id is wrong, the process restarted, or the retained status record expired after `server.http.retention`.
Diagnostic:
```sh
curl -i http://127.0.0.1:8080/runs/<run-id>
rg -n 'retention:' <config-path>
```
Safe fix: use the exact `run_id` returned by upload admission. Increase retention if operators need a longer status window.
Reference: [Operations](operations.md#http-upload-operation).

View File

@@ -1,27 +1,25 @@
pipelines:
- id: example-shared-root-source
- id: example-additive-source
source:
backend: local
path: examples/source-bundle
destinations:
- id: shared-local-root
- id: catalog-source
backend: local
path: workspace/published/shared-root
state:
mode: shared_root
path: workspace/published/additive-workflow
workflow: additive
publish:
source: true
html: false
- id: example-shared-root-html
- id: example-additive-html
source:
backend: local
path: examples/source-bundle
destinations:
- id: shared-local-root
- id: catalog-html
backend: local
path: workspace/published/shared-root
state:
mode: shared_root
path: workspace/published/additive-workflow
workflow: additive
publish:
source: false
html: true

View File

@@ -7,6 +7,7 @@ pipelines:
- id: local-source-archive
backend: local
path: workspace/published/archive-and-latest/archive
workflow: additive
path_mapping:
mode: preserve_relative
publish:
@@ -15,6 +16,7 @@ pipelines:
- id: local-html-latest
backend: local
path: workspace/published/archive-and-latest/latest
workflow: replacement
path_mapping:
mode: fixed
links:

View File

@@ -1,18 +1,19 @@
pipelines:
- id: example-merge-reconciliation
- id: example-replacement-workflow
source:
backend: local
path: examples/source-bundle
destinations:
- id: local-merge-html
- id: local-latest-html
backend: local
path: workspace/published/merge-reconciliation
path: workspace/published/replacement-workflow
workflow: replacement
path_mapping:
mode: fixed
publish:
source: false
html: true
transform:
markdown_to_html:
enabled: true
mode: sidecar
reconciliation:
mode: merge
mode: index

View File

@@ -170,7 +170,7 @@ func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, BackendName, logicalPath, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", written, opts.Size))
}
if opts.PreferAtomic {
if err := b.client.Rename(writePath, nativePath); err != nil {
if err := renamePromotedFile(b.client, writePath, nativePath, opts.Overwrite); err != nil {
return storage.Entry{}, b.translateError(storage.OpWriteFrom, logicalPath, err)
}
cleanup = false
@@ -178,6 +178,27 @@ func (b *Backend) WriteFrom(ctx context.Context, logicalPath string, r io.Reader
return b.Stat(ctx, logicalPath)
}
type sftpRenamer interface {
PosixRename(oldname, newname string) error
Rename(oldname, newname string) error
Remove(path string) error
}
func renamePromotedFile(client sftpRenamer, oldname, newname string, overwrite bool) error {
if !overwrite {
return client.Rename(oldname, newname)
}
if err := client.PosixRename(oldname, newname); err == nil {
return nil
} else if !isReplaceRenameFallbackError(err) {
return err
}
if err := client.Remove(newname); err != nil && !isNotExist(err) {
return err
}
return client.Rename(oldname, newname)
}
func (b *Backend) Stat(ctx context.Context, logicalPath string) (storage.Entry, error) {
if err := ctx.Err(); err != nil {
return storage.Entry{}, err
@@ -464,6 +485,14 @@ func isNotExist(err error) bool {
return errors.Is(err, fs.ErrNotExist) || errors.Is(err, os.ErrNotExist) || errors.Is(err, sftp.ErrSSHFxNoSuchFile)
}
func isReplaceRenameFallbackError(err error) bool {
if errors.Is(err, sftp.ErrSSHFxFailure) || errors.Is(err, sftp.ErrSSHFxOpUnsupported) {
return true
}
var statusErr *sftp.StatusError
return errors.As(err, &statusErr) && (statusErr.FxCode() == sftp.ErrSSHFxFailure || statusErr.FxCode() == sftp.ErrSSHFxOpUnsupported)
}
func (b *Backend) translateError(op, logicalPath string, err error) error {
kind := storage.ErrUnknown
switch {

View File

@@ -0,0 +1,125 @@
package ssh
import (
"errors"
"os"
"testing"
"github.com/pkg/sftp"
)
func TestRenamePromotedFileUsesPlainRenameWithoutOverwrite(t *testing.T) {
client := &recordingRenamer{}
if err := renamePromotedFile(client, "temp", "index.html", false); err != nil {
t.Fatalf("renamePromotedFile() error = %v", err)
}
if got, want := client.calls, []string{"rename temp index.html"}; !equalStrings(got, want) {
t.Fatalf("calls = %q, want %q", got, want)
}
}
func TestRenamePromotedFileUsesPosixRenameForOverwrite(t *testing.T) {
client := &recordingRenamer{}
if err := renamePromotedFile(client, "temp", "index.html", true); err != nil {
t.Fatalf("renamePromotedFile() error = %v", err)
}
if got, want := client.calls, []string{"posix temp index.html"}; !equalStrings(got, want) {
t.Fatalf("calls = %q, want %q", got, want)
}
}
func TestRenamePromotedFileFallsBackWhenReplaceRenameUnsupported(t *testing.T) {
for _, err := range []error{
sftp.ErrSSHFxOpUnsupported,
sftp.ErrSSHFxFailure,
&sftp.StatusError{Code: uint32(sftp.ErrSSHFxOpUnsupported)},
&sftp.StatusError{Code: uint32(sftp.ErrSSHFxFailure)},
} {
t.Run(err.Error(), func(t *testing.T) {
client := &recordingRenamer{posixErr: err}
if err := renamePromotedFile(client, "temp", "index.html", true); err != nil {
t.Fatalf("renamePromotedFile() error = %v", err)
}
want := []string{"posix temp index.html", "remove index.html", "rename temp index.html"}
if got := client.calls; !equalStrings(got, want) {
t.Fatalf("calls = %q, want %q", got, want)
}
})
}
}
func TestRenamePromotedFileIgnoresMissingTargetDuringFallback(t *testing.T) {
client := &recordingRenamer{
posixErr: sftp.ErrSSHFxOpUnsupported,
removeErr: &os.PathError{
Op: "remove",
Path: "index.html",
Err: os.ErrNotExist,
},
}
if err := renamePromotedFile(client, "temp", "index.html", true); err != nil {
t.Fatalf("renamePromotedFile() error = %v", err)
}
want := []string{"posix temp index.html", "remove index.html", "rename temp index.html"}
if got := client.calls; !equalStrings(got, want) {
t.Fatalf("calls = %q, want %q", got, want)
}
}
func TestRenamePromotedFileDoesNotFallbackForPermissionError(t *testing.T) {
client := &recordingRenamer{posixErr: sftp.ErrSSHFxPermissionDenied}
if err := renamePromotedFile(client, "temp", "index.html", true); !errors.Is(err, sftp.ErrSSHFxPermissionDenied) {
t.Fatalf("renamePromotedFile() error = %v, want permission denied", err)
}
if got, want := client.calls, []string{"posix temp index.html"}; !equalStrings(got, want) {
t.Fatalf("calls = %q, want %q", got, want)
}
}
func TestRenamePromotedFileReturnsRemoveFallbackError(t *testing.T) {
client := &recordingRenamer{
posixErr: sftp.ErrSSHFxOpUnsupported,
removeErr: sftp.ErrSSHFxPermissionDenied,
}
if err := renamePromotedFile(client, "temp", "index.html", true); !errors.Is(err, sftp.ErrSSHFxPermissionDenied) {
t.Fatalf("renamePromotedFile() error = %v, want permission denied", err)
}
want := []string{"posix temp index.html", "remove index.html"}
if got := client.calls; !equalStrings(got, want) {
t.Fatalf("calls = %q, want %q", got, want)
}
}
type recordingRenamer struct {
calls []string
posixErr error
renameErr error
removeErr error
}
func (r *recordingRenamer) PosixRename(oldname, newname string) error {
r.calls = append(r.calls, "posix "+oldname+" "+newname)
return r.posixErr
}
func (r *recordingRenamer) Rename(oldname, newname string) error {
r.calls = append(r.calls, "rename "+oldname+" "+newname)
return r.renameErr
}
func (r *recordingRenamer) Remove(path string) error {
r.calls = append(r.calls, "remove "+path)
return r.removeErr
}
func equalStrings(a, b []string) bool {
if len(a) != len(b) {
return false
}
for index := range a {
if a[index] != b[index] {
return false
}
}
return true
}

View File

@@ -211,29 +211,18 @@ func removePrunedStateRecords(ctx context.Context, backend storage.Backend, stat
if len(paths) == 0 {
return false, nil
}
if document.SingleOwner != nil {
next, changed := state.RemoveMissingOutputs(*document.SingleOwner, paths)
if document.Catalog != nil {
next, changed := state.RemoveMissingCatalogOwnerOutputs(*document.Catalog, scope, paths)
if !changed {
return false, nil
}
next.UpdatedAt = now
if err := state.Validate(next); err != nil {
if err := state.ValidateCatalog(next); err != nil {
return false, err
}
return true, writeRepairedState(ctx, backend, statePath, next)
}
if document.SharedRoot != nil {
next, changed := state.RemoveMissingSharedRootOwnerOutputs(*document.SharedRoot, scope, paths)
if !changed {
return false, nil
}
next.UpdatedAt = now
if err := state.ValidateSharedRoot(next); err != nil {
return false, err
}
return true, writeRepairedState(ctx, backend, statePath, next)
}
return false, fmt.Errorf("destination state document is empty")
return false, unsupportedStateDocumentError(document)
}
func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options PrunePlanOptions) (PrunePlanReport, error) {
@@ -266,17 +255,17 @@ func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options
}
func pruneCandidatesForDocument(document state.StateDocument, scope state.OwnerScope) ([]state.PruneCandidate, error) {
if document.SingleOwner != nil {
singleOwner := *document.SingleOwner
if singleOwner.PipelineID != scope.PipelineID || singleOwner.DestinationID != scope.DestinationID {
return nil, fmt.Errorf("state owner is %s/%s, not %s/%s", singleOwner.PipelineID, singleOwner.DestinationID, scope.PipelineID, scope.DestinationID)
if document.Catalog != nil {
return state.CatalogPruneCandidates(*document.Catalog, scope), nil
}
return state.SingleOwnerPruneCandidates(singleOwner), nil
return nil, unsupportedStateDocumentError(document)
}
if document.SharedRoot != nil {
return state.SharedRootPruneCandidates(*document.SharedRoot, scope), nil
func unsupportedStateDocumentError(document state.StateDocument) error {
if document.SupersededLegacy != nil {
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
}
return nil, fmt.Errorf("destination state document is empty")
return fmt.Errorf("destination state document is empty")
}
func pruneOlderThan(policy config.PrunePolicy) *time.Duration {

View File

@@ -15,7 +15,7 @@ import (
)
func TestPlanPruneDisabledPolicy(t *testing.T) {
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
document := state.StateDocument{Catalog: &state.CatalogState{}}
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
PipelineID: "reports",
DestinationID: "archive",
@@ -28,12 +28,12 @@ func TestPlanPruneDisabledPolicy(t *testing.T) {
}
}
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
func TestPlanPruneCatalogOutputs(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
olderThan := config.Duration(48 * time.Hour)
destinationState := pruneSingleOwnerState(now)
catalog := pruneCatalogState(now)
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
Enabled: true,
OlderThan: &olderThan,
}, PrunePlanOptions{
@@ -52,12 +52,12 @@ func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
}
}
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
func TestPlanPruneCatalogCurrentOwnerOnly(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
keepLatest := 0
sharedRoot := pruneSharedRootState(now)
catalog := pruneCatalogState(now)
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
Enabled: true,
KeepLatest: &keepLatest,
}, PrunePlanOptions{
@@ -68,10 +68,10 @@ func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
if err != nil {
t.Fatalf("PlanPrune() error = %v", err)
}
if got, want := report.CheckedCount, 1; got != want {
if got, want := report.CheckedCount, 2; got != want {
t.Fatalf("checked count = %d, want %d", got, want)
}
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt,fresh.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
}
@@ -80,8 +80,8 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
original := pruneSingleOwnerState(now)
writeFakeSingleOwnerStateForPrune(t, backend, original)
original := pruneCatalogState(now)
writeFakeCatalogState(t, backend, original)
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -102,12 +102,12 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "old.txt,fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
if !catalog.UpdatedAt.Equal(original.UpdatedAt) {
t.Fatalf("state updated_at = %s, want original %s", catalog.UpdatedAt, original.UpdatedAt)
}
}
@@ -115,7 +115,7 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -134,14 +134,18 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
}
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
assertFakeStateExists(t, backend)
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want fresh.txt", got)
}
if !destinationState.UpdatedAt.Equal(now) {
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
if catalog.SchemaVersion != state.CatalogSchemaVersion {
t.Fatalf("state schema_version = %d, want %d", catalog.SchemaVersion, state.CatalogSchemaVersion)
}
if !catalog.UpdatedAt.Equal(now) {
t.Fatalf("state updated_at = %s, want %s", catalog.UpdatedAt, now)
}
}
@@ -150,7 +154,7 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
backend := fake.New()
keepLatest := 0
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -170,18 +174,17 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
assertFakeStateExists(t, backend)
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want only failed output preserved", got)
}
}
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
func TestPrunePreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
keepLatest := 0
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -192,93 +195,56 @@ func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.
if err != nil {
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
}
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
t.Fatalf("deleted outputs = %q, want %q", got, want)
}
testutil.AssertFakeMissing(t, backend, "archive.txt")
testutil.AssertFakeFile(t, backend, "html.txt", "old")
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
sharedRoot := readFakeSharedRootStateForApp(t, backend)
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
}
}
func pruneSingleOwnerState(now time.Time) state.DistributorState {
func pruneCatalogState(now time.Time) state.CatalogState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
publishedAt := now.Add(-96 * time.Hour)
return state.DistributorState{
SchemaVersion: state.SchemaVersion,
createdAt := now.Add(-96 * time.Hour)
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
return state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: "test",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: []state.CatalogOutputFile{{
Path: "old.txt",
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
DistributorVersion: "test",
Outputs: []state.OutputFile{{
Path: "old.txt",
Source: source,
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}, {
Path: "fresh.txt",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SourcePath: "summary.txt",
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
CreatedAt: now.Add(-24 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
}},
}
}
func pruneSharedRootState(now time.Time) state.SharedRootState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
archive := state.CurrentOwnerScope("reports", "archive")
html := state.CurrentOwnerScope("reports", "html")
return state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: "test",
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{{
Scope: archive,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
}, {
Scope: html,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
}},
Outputs: []state.SharedRootOutputFile{{
Path: "archive.txt",
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
Owner: archive,
SourceID: manifest.ID,
SourceDigest: manifest.Digest,
SourceCreated: manifest.Created,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
CreatedAt: now.Add(-24 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
}, {
Path: "html.txt",
PipelineID: "reports",
DestinationID: "html",
Source: source,
Kind: state.OutputKindSource,
SourcePath: "summary.txt",
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
Owner: html,
SourceID: manifest.ID,
SourceDigest: manifest.Digest,
SourceCreated: manifest.Created,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}},
@@ -319,18 +285,31 @@ func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
}
}
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
func writeFakeCatalogState(t *testing.T, backend *fake.Backend, catalog state.CatalogState) {
t.Helper()
data, err := json.MarshalIndent(destinationState, "", " ")
data, err := json.MarshalIndent(catalog, "", " ")
if err != nil {
t.Fatalf("marshal single-owner state: %v", err)
t.Fatalf("marshal catalog state: %v", err)
}
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
for _, output := range destinationState.Outputs {
for _, output := range catalog.Outputs {
testutil.WriteFakeFile(t, backend, output.Path, "managed")
}
}
func readFakeCatalogState(t *testing.T, backend *fake.Backend) state.CatalogState {
t.Helper()
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
if err != nil {
t.Fatalf("read catalog state: %v", err)
}
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse catalog state: %v", err)
}
return catalog
}
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
t.Helper()
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {

View File

@@ -158,62 +158,25 @@ func buildReconcileStateReport(ctx context.Context, backend storage.Backend, pip
DryRun: options.DryRun,
}
scope := state.CurrentOwnerScope(pipeline.ID, destination.ID)
if document.SingleOwner != nil {
return reconcileSingleOwnerState(ctx, backend, statePath, *document.SingleOwner, scope, report, options)
if document.Catalog != nil {
return reconcileCatalogState(ctx, backend, statePath, *document.Catalog, scope, report, options)
}
return reconcileSharedRootState(ctx, backend, statePath, *document.SharedRoot, scope, report, options)
return ReconcileStateReport{}, unsupportedStateDocumentError(document)
}
func reconcileSingleOwnerState(ctx context.Context, backend storage.Backend, statePath string, destinationState state.DistributorState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
if destinationState.PipelineID != scope.PipelineID || destinationState.DestinationID != scope.DestinationID {
return ReconcileStateReport{}, fmt.Errorf("state owner is %s/%s, not %s/%s", destinationState.PipelineID, destinationState.DestinationID, scope.PipelineID, scope.DestinationID)
}
report.StateSchema = destinationState.SchemaVersion
report.OwnerScope = &ReconcileStateOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID}
managed := state.ManagedOutputPaths(destinationState)
missing, err := missingSingleOwnerOutputs(ctx, backend, destinationState.Outputs)
if err != nil {
return ReconcileStateReport{}, err
}
report.CheckedCount = len(managed)
report.MissingManagedOutputs = missing
unmanaged, err := unmanagedEntries(ctx, backend, managed)
if err != nil {
return ReconcileStateReport{}, err
}
report.UnmanagedEntries = unmanaged
report.WouldChange = options.DryRun && len(missing) > 0
if !options.DryRun && len(missing) > 0 {
missingPaths := missingReportPaths(missing)
next, changed := state.RemoveMissingOutputs(destinationState, missingPaths)
report.Changed = changed
if changed {
next.UpdatedAt = time.Now().UTC()
if err := state.Validate(next); err != nil {
return ReconcileStateReport{}, err
}
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
return ReconcileStateReport{}, err
}
}
}
return report, nil
}
func reconcileSharedRootState(ctx context.Context, backend storage.Backend, statePath string, sharedRoot state.SharedRootState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
report.StateSchema = sharedRoot.SchemaVersion
func reconcileCatalogState(ctx context.Context, backend storage.Backend, statePath string, catalog state.CatalogState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
report.StateSchema = catalog.SchemaVersion
report.OwnerScope = &ReconcileStateOwnerScope{
PipelineID: scope.PipelineID,
DestinationID: scope.DestinationID,
AllOwners: options.AllOwners,
}
managed := sharedRoot.AllManagedOutputPaths()
outputs := sharedRoot.Outputs
managed := state.CatalogManagedOutputPaths(catalog)
outputs := catalog.Outputs
if !options.AllOwners {
outputs = sharedRootOutputsForOwner(sharedRoot.Outputs, scope)
outputs = state.CatalogOutputsForOwner(catalog.Outputs, scope)
}
missing, err := missingSharedRootOutputs(ctx, backend, outputs)
missing, err := missingCatalogOutputs(ctx, backend, outputs)
if err != nil {
return ReconcileStateReport{}, err
}
@@ -228,17 +191,17 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
if !options.DryRun && len(missing) > 0 {
missingPaths := missingReportPaths(missing)
var next state.SharedRootState
var next state.CatalogState
var changed bool
if options.AllOwners {
next, changed = state.RemoveMissingSharedRootOutputs(sharedRoot, missingPaths)
next, changed = state.RemoveMissingCatalogOutputs(catalog, missingPaths)
} else {
next, changed = state.RemoveMissingSharedRootOwnerOutputs(sharedRoot, scope, missingPaths)
next, changed = state.RemoveMissingCatalogOwnerOutputs(catalog, scope, missingPaths)
}
report.Changed = changed
if changed {
next.UpdatedAt = time.Now().UTC()
if err := state.ValidateSharedRoot(next); err != nil {
if err := state.ValidateCatalog(next); err != nil {
return ReconcileStateReport{}, err
}
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
@@ -249,21 +212,7 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
return report, nil
}
func missingSingleOwnerOutputs(ctx context.Context, backend storage.Backend, outputs []state.OutputFile) ([]ReconcileStatePath, error) {
missing := make([]ReconcileStatePath, 0)
for _, output := range outputs {
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
if storage.IsNotFound(err) {
missing = append(missing, ReconcileStatePath{Path: output.Path, StorageStatus: "missing"})
continue
}
return nil, err
}
}
return missing, nil
}
func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outputs []state.SharedRootOutputFile) ([]ReconcileStatePath, error) {
func missingCatalogOutputs(ctx context.Context, backend storage.Backend, outputs []state.CatalogOutputFile) ([]ReconcileStatePath, error) {
missing := make([]ReconcileStatePath, 0)
for _, output := range outputs {
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
@@ -271,8 +220,8 @@ func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outp
missing = append(missing, ReconcileStatePath{
Path: output.Path,
OwnerScope: &ReconcileStateOwnerScope{
PipelineID: output.Owner.PipelineID,
DestinationID: output.Owner.DestinationID,
PipelineID: output.PipelineID,
DestinationID: output.DestinationID,
},
StorageStatus: "missing",
})
@@ -320,16 +269,6 @@ func unmanagedEntries(ctx context.Context, backend storage.Backend, managedPaths
return entries, nil
}
func sharedRootOutputsForOwner(outputs []state.SharedRootOutputFile, scope state.OwnerScope) []state.SharedRootOutputFile {
selected := make([]state.SharedRootOutputFile, 0, len(outputs))
for _, output := range outputs {
if output.Owner == scope {
selected = append(selected, output)
}
}
return selected
}
func missingReportPaths(missing []ReconcileStatePath) []string {
paths := make([]string, 0, len(missing))
for _, item := range missing {

View File

@@ -2,7 +2,6 @@ package app
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
@@ -17,9 +16,9 @@ import (
func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testing.T) {
backend := fake.New()
cfg := reconcileStateS3Config(t)
manifest := testutil.ValidManifest(testutil.BundleOptions{})
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
catalog := pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC))
writeFakeCatalogState(t, backend, catalog)
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
t.Fatalf("delete managed output: %v", err)
}
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
@@ -35,14 +34,14 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
if !report.WouldChange || report.Changed {
t.Fatalf("report changed=%t would_change=%t, want dry-run pending change", report.Changed, report.WouldChange)
}
if got := reportPathList(report.MissingManagedOutputs); got != "summary.txt" {
t.Fatalf("missing outputs = %q, want summary.txt", got)
if got := reportPathList(report.MissingManagedOutputs); got != "fresh.txt" {
t.Fatalf("missing outputs = %q, want fresh.txt", got)
}
if got := entryPathList(report.UnmanagedEntries); got != "extra.txt" {
t.Fatalf("unmanaged entries = %q, want extra.txt", got)
}
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
repaired := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
}
@@ -50,9 +49,8 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *testing.T) {
backend := fake.New()
cfg := reconcileStateS3Config(t)
manifest := testutil.ValidManifest(testutil.BundleOptions{})
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
t.Fatalf("delete managed output: %v", err)
}
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
@@ -67,12 +65,15 @@ func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *t
if !report.Changed || report.WouldChange {
t.Fatalf("report changed=%t would_change=%t, want applied change", report.Changed, report.WouldChange)
}
destinationState := readFakeSingleOwnerState(t, backend)
if err := state.Validate(destinationState); err != nil {
t.Fatalf("Validate() repaired state error = %v", err)
repaired := readFakeCatalogState(t, backend)
if err := state.ValidateCatalog(repaired); err != nil {
t.Fatalf("ValidateCatalog() repaired state error = %v", err)
}
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
t.Fatalf("state outputs = %q, want report.md", got)
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,html.txt" {
t.Fatalf("state outputs = %q, want old.txt,html.txt", got)
}
if repaired.SchemaVersion != state.CatalogSchemaVersion {
t.Fatalf("state schema_version = %d, want %d", repaired.SchemaVersion, state.CatalogSchemaVersion)
}
testutil.AssertFakeFile(t, backend, "extra.txt", "unmanaged")
}
@@ -99,12 +100,11 @@ func TestReconcileStateInvalidStateFailsWithoutRewrite(t *testing.T) {
}
}
func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
func TestReconcileStateOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
backend := fake.New()
cfg := reconcileStateS3Config(t)
sharedRoot := reconcileSharedRootFixture(t)
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
t.Fatalf("delete managed outputs: %v", err)
}
@@ -118,18 +118,17 @@ func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T)
if !report.Changed {
t.Fatal("report changed = false, want true")
}
repaired := readFakeSharedRootStateForApp(t, backend)
if got := strings.Join(repaired.AllManagedOutputPaths(), ","); got != "report.html" {
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
repaired := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt,html.txt" {
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
}
}
func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
func TestReconcileStateAllOwnersRepairsEveryOwner(t *testing.T) {
backend := fake.New()
cfg := reconcileStateS3Config(t)
sharedRoot := reconcileSharedRootFixture(t)
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
t.Fatalf("delete managed outputs: %v", err)
}
@@ -141,12 +140,12 @@ func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
if err != nil {
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
}
if !report.Changed || report.CheckedCount != 2 {
if !report.Changed || report.CheckedCount != 3 {
t.Fatalf("report changed=%t checked=%d, want all-owner repair", report.Changed, report.CheckedCount)
}
repaired := readFakeSharedRootStateForApp(t, backend)
if got := repaired.AllManagedOutputPaths(); len(got) != 0 {
t.Fatalf("shared-root outputs = %#v, want none", got)
repaired := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt" {
t.Fatalf("catalog outputs = %q, want fresh.txt", got)
}
}
@@ -165,93 +164,6 @@ func reconcileStateS3Config(t *testing.T) config.Config {
return cfg
}
func readFakeSingleOwnerState(t *testing.T, backend *fake.Backend) state.DistributorState {
t.Helper()
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
if err != nil {
t.Fatalf("read state: %v", err)
}
destinationState, err := state.Parse(data)
if err != nil {
t.Fatalf("parse state: %v", err)
}
return destinationState
}
func writeFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend, sharedRoot state.SharedRootState) {
t.Helper()
data, err := json.MarshalIndent(sharedRoot, "", " ")
if err != nil {
t.Fatalf("marshal shared-root state: %v", err)
}
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
for _, output := range sharedRoot.Outputs {
testutil.WriteFakeFile(t, backend, output.Path, "old")
}
}
func readFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend) state.SharedRootState {
t.Helper()
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
if err != nil {
t.Fatalf("read shared-root state: %v", err)
}
sharedRoot, err := state.ParseSharedRoot(data)
if err != nil {
t.Fatalf("parse shared-root state: %v", err)
}
return sharedRoot
}
func reconcileSharedRootFixture(t *testing.T) state.SharedRootState {
t.Helper()
source := testutil.ValidManifest(testutil.BundleOptions{})
htmlSource := source
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
return state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: "test",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{{
Scope: state.CurrentOwnerScope("reports", "archive"),
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: source},
}, {
Scope: state.CurrentOwnerScope("reports", "html"),
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
Source: state.SourceState{Manifest: htmlSource},
}},
Outputs: []state.SharedRootOutputFile{{
Path: "report.md",
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
Owner: state.CurrentOwnerScope("reports", "archive"),
SourceID: source.ID,
SourceDigest: source.Digest,
SourceCreated: source.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}, {
Path: "report.html",
Kind: state.OutputKindGenerated,
SourcePath: "report.md",
Transform: "markdown_to_html",
SHA256: "sha256:" + strings.Repeat("a", 64),
Size: 128,
Owner: state.CurrentOwnerScope("reports", "html"),
SourceID: htmlSource.ID,
SourceDigest: htmlSource.Digest,
SourceCreated: htmlSource.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}},
}
}
func reportPathList(paths []ReconcileStatePath) string {
values := make([]string, 0, len(paths))
for _, path := range paths {

View File

@@ -69,10 +69,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
Publish: *request.destination.Publish,
Transform: request.destination.Transform,
Links: request.destination.Links,
State: request.destination.State,
Reconciliation: request.destination.Reconciliation,
Workflow: request.destination.Workflow,
Transformers: request.transforms,
Transfer: request.destination.Transfer,
DistributorVersion: Version,
Force: request.options.Force,
}
@@ -82,8 +80,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
}
if isFixedPathDestination(request.destination) {
plan.PathMapping = config.PathMappingFixed
if request.options.DryRun && isDestructiveFixedPathAction(plan.Action) {
warning := fixedPathReplacementWarning(plan)
if request.options.DryRun && isFixedPathWorkflowAction(plan.Action) {
warning := fixedPathWorkflowWarning(plan)
request.recorder.addPipelineWarning(request.pipelineIndex, warning)
}
}
@@ -141,6 +139,7 @@ func (recorder *runReportRecorder) recordDestinationFailure(pipelineIndex int, f
recorder.failures.add(failure.pipelineID, failure.destinationID, failure.backend, storage.DisplayPath(failure.bundlePath), failure.err)
recorder.summary.recordFailure()
if includeAction {
recorder.summary.recordFailureAction(action.Action)
recorder.addPipelineAction(pipelineIndex, action)
}
}
@@ -161,5 +160,8 @@ func completePlanIdentity(plan publish.Plan, pipeline config.Pipeline, destinati
if plan.DestinationBundlePath == "" {
plan.DestinationBundlePath = selection.DestinationBundlePath
}
if plan.Workflow == "" {
plan.Workflow = destination.Workflow
}
return plan
}

View File

@@ -6,20 +6,19 @@ import (
)
func shouldNotify(action publish.Action) bool {
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
return action == publish.ActionPublishNew || action == publish.ActionUpsertAdditive || action == publish.ActionReplaceCatalog || action == publish.ActionForceReplace
}
func notifyEvent(plan publish.Plan) notify.Event {
outputs := make([]notify.Output, 0, len(plan.Outputs))
for _, output := range plan.Outputs {
stateOutput := output.StateOutputFile()
outputs = append(outputs, notify.Output{
Path: stateOutput.Path,
Kind: stateOutput.Kind,
SourcePath: stateOutput.SourcePath,
Transform: stateOutput.Transform,
SHA256: stateOutput.SHA256,
Size: stateOutput.Size,
Path: output.DestinationPath,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
SHA256: output.SHA256,
Size: output.Size,
})
}
return notify.Event{

View File

@@ -60,7 +60,7 @@ func writeRunActionLine(w io.Writer, action RunActionRecord) {
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=error reason=%q\n", action.BundlePath, destinationID, action.Backend, pathMappingRecordSummary(action), action.Reason)
return
}
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, outputRecordSummary(action.Outputs), action.Reason)
fmt.Fprintf(w, " - bundle=%s destination=%s backend=%s%s action=%s%s outputs=%s reason=%q\n", action.BundlePath, action.DestinationID, action.Backend, pathMappingRecordSummary(action), action.Action, workflowRecordSummary(action), outputRecordSummary(action.Outputs), action.Reason)
}
func pathMappingRecordSummary(action RunActionRecord) string {
@@ -70,6 +70,13 @@ func pathMappingRecordSummary(action RunActionRecord) string {
return fmt.Sprintf(" path_mapping=fixed target=%s", action.DestinationPath)
}
func workflowRecordSummary(action RunActionRecord) string {
if action.Workflow == "" {
return ""
}
return fmt.Sprintf(" workflow=%s", action.Workflow)
}
func outputRecordSummary(outputs []RunOutputRecord) string {
if len(outputs) == 0 {
return "none"
@@ -136,6 +143,7 @@ type RunActionRecord struct {
BundlePath string `json:"bundle_path"`
DestinationPath string `json:"destination_path"`
PathMapping string `json:"path_mapping,omitempty"`
Workflow string `json:"workflow,omitempty"`
Action string `json:"action"`
PrimaryURL string `json:"primary_url,omitempty"`
Reason string `json:"reason,omitempty"`
@@ -158,6 +166,13 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
if destinationID == "" {
destinationID = "unknown"
}
action := "error"
outputs := []RunOutputRecord{}
switch plan.Action {
case publish.ActionFailUnmanaged, publish.ActionFailConflict:
action = string(plan.Action)
outputs = runOutputsFromPlan(plan.Outputs)
}
return RunActionRecord{
PipelineID: plan.PipelineID,
DestinationID: destinationID,
@@ -166,10 +181,11 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
BundlePath: storage.DisplayPath(plan.BundlePath),
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
PathMapping: plan.PathMapping,
Action: "error",
Workflow: plan.Workflow,
Action: action,
PrimaryURL: plan.PrimaryURL,
Reason: planErr.Error(),
Outputs: []RunOutputRecord{},
Outputs: outputs,
}
}
return RunActionRecord{
@@ -180,6 +196,7 @@ func runActionFromPlan(backend string, plan publish.Plan, planErr error) RunActi
BundlePath: storage.DisplayPath(plan.BundlePath),
DestinationPath: storage.DisplayPath(plan.DestinationBundlePath),
PathMapping: plan.PathMapping,
Workflow: plan.Workflow,
Action: string(plan.Action),
PrimaryURL: plan.PrimaryURL,
Reason: plan.Reason,
@@ -203,15 +220,14 @@ func errorAction(pipelineID, destinationID, backend, bundlePath string, err erro
func runOutputsFromPlan(outputs []publish.Output) []RunOutputRecord {
results := make([]RunOutputRecord, 0, len(outputs))
for _, output := range outputs {
stateOutput := output.StateOutputFile()
results = append(results, RunOutputRecord{
Path: stateOutput.Path,
Kind: stateOutput.Kind,
SourcePath: stateOutput.SourcePath,
Transform: stateOutput.Transform,
URL: stateOutput.URL,
SHA256: stateOutput.SHA256,
Size: stateOutput.Size,
Path: output.DestinationPath,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
})
}
return results

View File

@@ -62,12 +62,22 @@ func fixedPathSelectionWarning(pipelineID, destinationID string, selections []de
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed candidates=%d selected_bundle=%s destination_bundle=.", pipelineID, destinationID, candidateCount, selected)}
}
func isDestructiveFixedPathAction(action publish.Action) bool {
return action == publish.ActionReplaceOlder || action == publish.ActionForceReplace
func isFixedPathWorkflowAction(action publish.Action) bool {
return action == publish.ActionUpsertAdditive || action == publish.ActionReplaceCatalog || action == publish.ActionForceReplace
}
func fixedPathReplacementWarning(plan publish.Plan) OutputWarning {
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed action=%s replaces destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Action, storage.DisplayPath(plan.BundlePath))}
func fixedPathWorkflowWarning(plan publish.Plan) OutputWarning {
switch plan.Action {
case publish.ActionReplaceCatalog:
if plan.ClearDestinationRoot {
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s clears destination root before writing selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
}
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s replaces current-owner catalog outputs for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
case publish.ActionUpsertAdditive:
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s upserts planned outputs at destination root for selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
default:
return OutputWarning{Message: fmt.Sprintf("pipeline=%s destination=%s path_mapping=fixed workflow=%s action=%s writes selected_bundle=%s", plan.PipelineID, plan.DestinationID, plan.Workflow, plan.Action, storage.DisplayPath(plan.BundlePath))}
}
}
func destinationIDs(destinations []config.Destination) []string {

View File

@@ -10,9 +10,12 @@ type runSummary struct {
dryRun bool
planned int
publishNew int
replaceOlder int
upsertAdditive int
replaceCatalog int
skipSame int
forceReplace int
skipped int
failUnmanaged int
failConflict int
failures int
fixedPath int
}
@@ -22,12 +25,23 @@ func (s *runSummary) recordPlan(action publish.Action) {
switch action {
case publish.ActionPublishNew:
s.publishNew++
case publish.ActionReplaceOlder:
s.replaceOlder++
case publish.ActionUpsertAdditive:
s.upsertAdditive++
case publish.ActionReplaceCatalog:
s.replaceCatalog++
case publish.ActionForceReplace:
s.forceReplace++
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
s.skipped++
case publish.ActionSkipSame:
s.skipSame++
}
}
func (s *runSummary) recordFailureAction(action string) {
switch action {
case string(publish.ActionFailUnmanaged):
s.failUnmanaged++
case string(publish.ActionFailConflict):
s.failConflict++
}
}
@@ -43,16 +57,19 @@ type RunSummaryCounters struct {
Status string `json:"status"`
Planned int `json:"planned"`
PublishNew int `json:"publish_new"`
ReplaceOlder int `json:"replace_older"`
UpsertAdditive int `json:"upsert_additive"`
ReplaceCatalog int `json:"replace_catalog"`
SkipSame int `json:"skip_same"`
ForceReplace int `json:"force_replace"`
Skipped int `json:"skipped"`
FailUnmanaged int `json:"fail_unmanaged"`
FailConflict int `json:"fail_conflict"`
Failed int `json:"failed"`
DryRun bool `json:"dry_run"`
FixedPath int `json:"fixed_path"`
}
func (s RunSummaryCounters) Line() string {
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d force_replace=%d skipped=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.ReplaceOlder, s.ForceReplace, s.Skipped, s.Failed, s.DryRun, s.FixedPath)
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d upsert_additive=%d replace_catalog=%d skip_same=%d force_replace=%d fail_unmanaged=%d fail_conflict=%d failed=%d dry_run=%t fixed_path=%d", s.Status, s.Planned, s.PublishNew, s.UpsertAdditive, s.ReplaceCatalog, s.SkipSame, s.ForceReplace, s.FailUnmanaged, s.FailConflict, s.Failed, s.DryRun, s.FixedPath)
}
func (s runSummary) Result() RunSummaryCounters {
@@ -64,9 +81,12 @@ func (s runSummary) Result() RunSummaryCounters {
Status: status,
Planned: s.planned,
PublishNew: s.publishNew,
ReplaceOlder: s.replaceOlder,
UpsertAdditive: s.upsertAdditive,
ReplaceCatalog: s.replaceCatalog,
SkipSame: s.skipSame,
ForceReplace: s.forceReplace,
Skipped: s.skipped,
FailUnmanaged: s.failUnmanaged,
FailConflict: s.failConflict,
Failed: s.failures,
DryRun: s.dryRun,
FixedPath: s.fixedPath,

View File

@@ -15,7 +15,6 @@ import (
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/notify"
"gitea.maximumdirect.net/eric/distributor/internal/publish"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
@@ -42,8 +41,8 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) {
for _, want := range []string{
"Configured pipelines: 1",
"- pipeline=reports source=local bundles=1 destinations=archive",
"bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt",
"Final status: ok planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true",
"bundle=. destination=archive backend=local action=publish_new workflow=additive outputs=report.md,summary.txt",
"Final status: ok planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true",
} {
if !strings.Contains(output, want) {
t.Fatalf("Run() output = %q, want substring %q", output, want)
@@ -211,75 +210,14 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
}
if destinationState.Source.Manifest.ID != manifest.ID {
t.Fatalf("state source id = %q, want %q", destinationState.Source.Manifest.ID, manifest.ID)
if destinationState.SourceID != manifest.ID {
t.Fatalf("state source id = %q, want %q", destinationState.SourceID, manifest.ID)
}
if got, want := len(destinationState.Outputs), 2; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
}
if destinationState.Links != nil || destinationState.Outputs[0].URL != "" {
t.Fatalf("state links = %#v output URL=%q, want absent", destinationState.Links, destinationState.Outputs[0].URL)
}
}
func TestRunSharedRootDryRunWritesNoOutputsOrState(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
Files: []testFile{{Path: "report.md", Data: "# Report\n"}},
})
configPath := writeSharedRootLocalConfig(t, sourceRoot, destinationRoot)
cfg, err := config.LoadFile(configPath)
if err != nil {
t.Fatalf("load config: %v", err)
}
report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment)
if err != nil {
t.Fatalf("dry-run buildRunReportWithBackendFactory() error = %v", err)
}
if got, want := report.Actions[0].Action, string(publish.ActionPublishNew); got != want {
t.Fatalf("dry-run action = %q, want %q", got, want)
}
if _, statErr := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(statErr) {
t.Fatalf("output stat error = %v, want absent", statErr)
}
if _, statErr := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(statErr) {
t.Fatalf("state file stat error = %v, want absent", statErr)
}
}
func TestRunPublishesTwoPipelinesIntoSharedRoot(t *testing.T) {
firstSourceRoot := t.TempDir()
secondSourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, firstSourceRoot, "", testBundleOptions{
ID: "reports.first",
Files: []testFile{{Path: "first.md", Data: "# First\n"}},
})
writeSourceBundle(t, secondSourceRoot, "", testBundleOptions{
ID: "reports.second",
Files: []testFile{{Path: "second.md", Data: "# Second\n"}},
})
err := Run(context.Background(), RunOptions{ConfigPath: writeTwoPipelineSharedRootConfig(t, firstSourceRoot, secondSourceRoot, destinationRoot)})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "first.md"), "# First\n")
testutil.AssertFile(t, filepath.Join(destinationRoot, "second.md"), "# Second\n")
destinationState := readSharedRootStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if got, want := len(destinationState.Owners), 2; got != want {
t.Fatalf("owner count = %d, want %d", got, want)
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-first", "archive")); !ok {
t.Fatal("reports-first/archive owner missing")
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-second", "archive")); !ok {
t.Fatal("reports-second/archive owner missing")
}
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "first.md,second.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
if destinationState.PrimaryURL != "" || destinationState.Outputs[0].URL != "" {
t.Fatalf("state primary URL = %q output URL=%q, want absent", destinationState.PrimaryURL, destinationState.Outputs[0].URL)
}
}
@@ -424,8 +362,8 @@ func TestRunRecordsLinksForNestedBundlePath(t *testing.T) {
t.Fatalf("Run() error = %v", err)
}
destinationState := readStateFile(t, filepath.Join(destinationRoot, "daily", "brentwood", storage.StateFileName))
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
t.Fatalf("state links = %#v, want source primary URL", destinationState.Links)
if destinationState.PrimaryURL != "https://reports.example.com/archive/daily/brentwood/report.md" {
t.Fatalf("state primary URL = %q, want source primary URL", destinationState.PrimaryURL)
}
outputs := outputsByPath(destinationState.Outputs)
if outputs["report.md"].URL != "https://reports.example.com/archive/daily/brentwood/report.md" {
@@ -449,8 +387,8 @@ func TestRunRecordsLinksForFixedIndexDestination(t *testing.T) {
t.Fatalf("Run() error = %v", err)
}
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if destinationState.Links == nil || destinationState.Links.PrimaryURL != "https://reports.example.com/latest/" {
t.Fatalf("state links = %#v, want fixed index primary URL", destinationState.Links)
if destinationState.PrimaryURL != "https://reports.example.com/latest/" {
t.Fatalf("state primary URL = %q, want fixed index primary URL", destinationState.PrimaryURL)
}
if got, want := len(destinationState.Outputs), 1; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
@@ -489,8 +427,8 @@ func TestRunFixedPathPublishesNewestBundleAtDestinationRoot(t *testing.T) {
t.Fatalf("nested new report stat error = %v, want not exist", err)
}
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if destinationState.Source.Manifest.ID != "reports.new" {
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
if destinationState.SourceID != "reports.new" {
t.Fatalf("state source id = %q, want reports.new", destinationState.SourceID)
}
}
@@ -519,8 +457,8 @@ func TestRunFixedPathTieBreaksByBundlePath(t *testing.T) {
t.Fatalf("Run() error = %v", err)
}
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if destinationState.Source.Manifest.ID != "reports.a" {
t.Fatalf("state source id = %q, want reports.a", destinationState.Source.Manifest.ID)
if destinationState.SourceID != "reports.a" {
t.Fatalf("state source id = %q, want reports.a", destinationState.SourceID)
}
}
@@ -568,7 +506,7 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
{Path: "summary.txt", Data: "Old summary\n"},
},
})
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingFixed, config.WorkflowReplacement)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
@@ -592,8 +530,9 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
}
output := stdout.String()
for _, want := range []string{
"Warning: pipeline=reports destination=archive path_mapping=fixed action=replace_older replaces destination root for selected_bundle=new",
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_older",
"Warning: pipeline=reports destination=archive path_mapping=fixed workflow=replacement action=replace_catalog replaces current-owner catalog outputs for selected_bundle=new",
"bundle=new destination=archive backend=local path_mapping=fixed target=. action=replace_catalog workflow=replacement outputs=report.md,summary.txt reason=\"\"",
"replace_catalog=1",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
@@ -602,7 +541,7 @@ func TestRunFixedPathDryRunWarnsForReplacement(t *testing.T) {
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
}
func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
func TestRunJSONIncludesWorkflowActionAndSummary(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "old", testBundleOptions{
@@ -613,12 +552,10 @@ func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
{Path: "summary.txt", Data: "Old summary\n"},
},
})
configPath := testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed)
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingFixed, config.WorkflowReplacement)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nOld.\n")
writeSourceBundle(t, sourceRoot, "new", testBundleOptions{
ID: "reports.new",
Created: testutil.DefaultCreated.Add(time.Hour),
@@ -628,44 +565,35 @@ func TestRunFixedPathReplacesOlderManagedState(t *testing.T) {
},
})
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("second Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if destinationState.Source.Manifest.ID != "reports.new" {
t.Fatalf("state source id = %q, want reports.new", destinationState.Source.Manifest.ID)
}
}
func TestRunFixedPathSkipsWhenDestinationStateIsNewer(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
newer := testutil.ValidManifest(testutil.BundleOptions{
ID: "reports.newer",
Created: testutil.DefaultCreated.Add(time.Hour),
})
writeDestinationState(t, destinationRoot, "", newer)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nExisting.\n"), 0o600); err != nil {
t.Fatalf("write existing report: %v", err)
}
writeSourceBundle(t, sourceRoot, "older", testBundleOptions{
ID: "reports.older",
Created: testutil.DefaultCreated,
})
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: testutil.WriteLocalConfigWithPathMapping(t, sourceRoot, destinationRoot, config.PathMappingFixed),
ConfigPath: configPath,
DryRun: true,
Stdout: &stdout,
OutputFormat: OutputFormatJSON,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
result := decodeAppResult(t, stdout.String())
actions, ok := result["actions"].([]any)
if !ok || len(actions) != 1 {
t.Fatalf("actions = %#v, want one action", result["actions"])
}
action, ok := actions[0].(map[string]any)
if !ok {
t.Fatalf("action = %#v, want object", actions[0])
}
if action["action"] != "replace_catalog" || action["workflow"] != "replacement" || action["reason"] != nil {
t.Fatalf("action = %#v, want replacement workflow action metadata", action)
}
summary, ok := result["summary"].(map[string]any)
if !ok {
t.Fatalf("summary = %#v, want object", result["summary"])
}
if summary["replace_catalog"] != float64(1) || summary["upsert_additive"] != float64(0) || summary["force_replace"] != float64(0) {
t.Fatalf("summary = %#v, want replacement workflow counter only", summary)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nExisting.\n")
}
func TestRunFixedPathFailsUnmanagedWithoutForce(t *testing.T) {
@@ -709,6 +637,10 @@ func TestRunFixedPathForceReplacementStaysWithinDestinationRoot(t *testing.T) {
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
t.Fatalf("unmanaged stat error = %v, want removed", err)
}
catalog := readLocalCatalogState(t, destinationRoot)
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
}
}
func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
@@ -721,14 +653,6 @@ func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
{Path: "summary.txt", Data: "Old summary\n"},
},
})
writeSourceBundle(t, localSourceRoot, "new", testBundleOptions{
ID: "reports.new",
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
s3Destination := fake.New()
sshDestination := fake.New()
cfg := config.Config{Pipelines: []config.Pipeline{{
@@ -760,6 +684,30 @@ func TestRunFixedPathRemoteBackendsUseBackendRoots(t *testing.T) {
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{}, provider); err != nil {
t.Fatalf("Run() error = %v", err)
}
testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nOld.\n")
testutil.AssertFakeFile(t, sshDestination, "summary.txt", "Old summary\n")
writeSourceBundle(t, localSourceRoot, "new", testBundleOptions{
ID: "reports.new",
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
var stdout bytes.Buffer
if err := runConfigWithBackendFactory(context.Background(), cfg, RunOptions{Stdout: &stdout}, provider); err != nil {
t.Fatalf("second Run() error = %v", err)
}
for _, want := range []string{
"destination=object-latest backend=s3 path_mapping=fixed target=. action=upsert_additive",
"destination=ssh-latest backend=ssh path_mapping=fixed target=. action=upsert_additive",
"planned=2",
} {
if !strings.Contains(stdout.String(), want) {
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
}
}
testutil.AssertFakeFile(t, s3Destination, "report.md", "# Report\nNew.\n")
testutil.AssertFakeFile(t, s3Destination, "summary.txt", "New summary\n")
testutil.AssertFakeMissing(t, s3Destination, "new/report.md")
@@ -828,17 +776,22 @@ func TestRunNotifiesGeneratedOutputMetadata(t *testing.T) {
func TestRunNotifiesAfterReplacement(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
older := manifest
older.Created = older.Created.Add(-time.Hour)
writeDestinationState(t, destinationRoot, "", older)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old output: %v", err)
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, config.WorkflowReplacement)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
notifier := &recordingNotifier{}
err := Run(context.Background(), RunOptions{
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
ConfigPath: configPath,
Notifier: notifier,
})
if err != nil {
@@ -847,55 +800,8 @@ func TestRunNotifiesAfterReplacement(t *testing.T) {
if got, want := len(notifier.events), 1; got != want {
t.Fatalf("notification count = %d, want %d", got, want)
}
if notifier.events[0].Action != "replace_older" {
t.Fatalf("notification action = %q, want replace_older", notifier.events[0].Action)
}
}
func TestRunMergeReconciliationRetainsManagedOutput(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
older := manifest
older.Created = older.Created.Add(-time.Hour)
defaultManifest := testutil.ValidManifest(testutil.BundleOptions{})
older.Files = append([]bundle.ManifestFile(nil), defaultManifest.Files...)
older.Digest = bundle.BundleDigest(older.Files)
writeDestinationState(t, destinationRoot, "", older)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old report: %v", err)
}
if err := os.WriteFile(filepath.Join(destinationRoot, "summary.txt"), []byte("old summary\n"), 0o600); err != nil {
t.Fatalf("write old summary: %v", err)
}
configPath := writeConfigFile(t, `
pipelines:
- id: reports
source:
backend: local
path: `+sourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
reconciliation:
mode: merge
`)
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "old summary\n")
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
if got, want := len(destinationState.Outputs), 2; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
if notifier.events[0].Action != "replace_catalog" {
t.Fatalf("notification action = %q, want replace_catalog", notifier.events[0].Action)
}
}
@@ -1002,8 +908,8 @@ func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
firstDestination := t.TempDir()
secondDestination := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged file: %v", err)
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged planned file: %v", err)
}
cfg, err := config.LoadFile(writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination))
if err != nil {
@@ -1020,8 +926,8 @@ func TestBuildRunReportIncludesPartialFailures(t *testing.T) {
if got, want := len(report.Actions), 2; got != want {
t.Fatalf("action count = %d, want %d", got, want)
}
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "error" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
t.Fatalf("first action = %#v, want archive-one error", report.Actions[0])
if report.Actions[0].DestinationID != "archive-one" || report.Actions[0].Action != "fail_unmanaged" || !strings.Contains(report.Actions[0].Reason, "fail_unmanaged") {
t.Fatalf("first action = %#v, want archive-one unmanaged failure", report.Actions[0])
}
if report.Actions[1].DestinationID != "archive-two" || report.Actions[1].Action != "publish_new" {
t.Fatalf("second action = %#v, want archive-two publish_new", report.Actions[1])
@@ -1156,7 +1062,7 @@ func TestRunStillRunsAllConfiguredPipelines(t *testing.T) {
testutil.AssertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
}
func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
func TestRunNotifiesForAdditiveUpsert(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
@@ -1164,14 +1070,24 @@ func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
Created: testutil.DefaultCreated.Add(time.Hour),
Files: []testFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "summary.txt", Data: "New summary\n"},
},
})
notifier := &recordingNotifier{}
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Notifier: notifier})
if err != nil {
t.Fatalf("second Run() error = %v", err)
}
if len(notifier.events) != 0 {
t.Fatalf("notifications = %#v, want none", notifier.events)
if got, want := len(notifier.events), 1; got != want {
t.Fatalf("notification count = %d, want %d", got, want)
}
if notifier.events[0].Action != "upsert_additive" {
t.Fatalf("notification action = %q, want upsert_additive", notifier.events[0].Action)
}
}
@@ -1202,8 +1118,8 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
firstDestination := t.TempDir()
secondDestination := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged file: %v", err)
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged planned file: %v", err)
}
var stdout bytes.Buffer
@@ -1219,9 +1135,9 @@ func TestRunContinuesAfterDestinationFailure(t *testing.T) {
}
output := stdout.String()
for _, want := range []string{
"destination=archive-one backend=local action=error",
"destination=archive-two backend=local action=publish_new",
"Final status: failed planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=1 dry_run=false",
"destination=archive-one backend=local action=fail_unmanaged workflow=additive",
"destination=archive-two backend=local action=publish_new workflow=additive",
"Final status: failed planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=1 fail_conflict=0 failed=1 dry_run=false",
} {
if !strings.Contains(output, want) {
t.Fatalf("stdout = %q, want substring %q", output, want)
@@ -1471,6 +1387,55 @@ func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
statePath := filepath.Join(destinationRoot, storage.StateFileName)
stateBefore, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("read state before second run: %v", err)
}
reportBefore, err := os.ReadFile(filepath.Join(destinationRoot, "report.md"))
if err != nil {
t.Fatalf("read report before second run: %v", err)
}
notifier := &recordingNotifier{}
var stdout bytes.Buffer
err = Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout, Notifier: notifier})
if err != nil {
t.Fatalf("second Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=skip_same") {
t.Fatalf("stdout = %q, want skip_same", stdout.String())
}
if !strings.Contains(stdout.String(), "Final status: ok planned=1 publish_new=0 upsert_additive=0 replace_catalog=0 skip_same=1") {
t.Fatalf("stdout = %q, want skip_same summary", stdout.String())
}
stateAfter, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("read state after second run: %v", err)
}
if string(stateAfter) != string(stateBefore) {
t.Fatalf("state changed during skip")
}
reportAfter, err := os.ReadFile(filepath.Join(destinationRoot, "report.md"))
if err != nil {
t.Fatalf("read report after second run: %v", err)
}
if string(reportAfter) != string(reportBefore) {
t.Fatalf("report changed during skip")
}
if got, want := len(notifier.events), 0; got != want {
t.Fatalf("notification count = %d, want %d", got, want)
}
}
func TestRunReplacementSkipsWhenDestinationStateMatches(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
configPath := writeLocalConfigWithWorkflow(t, sourceRoot, destinationRoot, config.PathMappingPreserveRelative, config.WorkflowReplacement)
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
t.Fatalf("first Run() error = %v", err)
}
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout})
@@ -1482,63 +1447,6 @@ func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
}
}
func TestRunReplacesOlderDestination(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
older := manifest
older.Created = older.Created.Add(-time.Hour)
writeDestinationState(t, destinationRoot, "", older)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
t.Fatalf("write old output: %v", err)
}
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=replace_older") {
t.Fatalf("stdout = %q, want replace_older", stdout.String())
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
}
func TestRunSkipsNewerDestination(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
newer := manifest
newer.Created = newer.Created.Add(time.Hour)
writeDestinationState(t, destinationRoot, "", newer)
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
t.Fatalf("write newer output: %v", err)
}
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
}
func TestRunFailsOnConflict(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
manifest.ID = "other.source"
writeDestinationState(t, destinationRoot, "", manifest)
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot)})
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
t.Fatalf("Run() error = %v, want fail_conflict", err)
}
}
func TestRunFailsOnUnmanagedDestination(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
@@ -1573,10 +1481,50 @@ func TestRunForceReplacesUnmanagedDestination(t *testing.T) {
if !strings.Contains(stdout.String(), "action=force_replace") {
t.Fatalf("stdout = %q, want force_replace", stdout.String())
}
if !strings.Contains(stdout.String(), "Final status: ok planned=1 publish_new=0 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=1 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=false") {
t.Fatalf("stdout = %q, want force_replace counter only", stdout.String())
}
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); !os.IsNotExist(err) {
t.Fatalf("unmanaged file stat error = %v, want not exist", err)
}
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
catalog := readLocalCatalogState(t, destinationRoot)
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
}
}
func TestRunForceReplacementJSONOutput(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("old"), 0o600); err != nil {
t.Fatalf("write unmanaged file: %v", err)
}
var stdout bytes.Buffer
err := Run(context.Background(), RunOptions{
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
Force: true,
Stdout: &stdout,
OutputFormat: OutputFormatJSON,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
result := decodeAppResult(t, stdout.String())
actions, ok := result["actions"].([]any)
if !ok || len(actions) != 1 {
t.Fatalf("actions = %#v, want one action", result["actions"])
}
action, ok := actions[0].(map[string]any)
if !ok || action["action"] != "force_replace" || action["workflow"] != "additive" {
t.Fatalf("action = %#v, want force_replace additive", actions[0])
}
summary, ok := result["summary"].(map[string]any)
if !ok || summary["force_replace"] != float64(1) || summary["publish_new"] != float64(0) {
t.Fatalf("summary = %#v, want force_replace only", result["summary"])
}
}
func TestRunFansOutToLocalDestinations(t *testing.T) {
@@ -1649,7 +1597,7 @@ func TestRunExercisesRemoteBackendShapesThroughCommonPath(t *testing.T) {
"pipeline=local-to-ssh source=local",
"destination=ssh-archive backend=ssh action=publish_new",
"pipeline=ssh-to-local source=ssh",
"Final status: ok planned=4 publish_new=4 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true",
"Final status: ok planned=4 publish_new=4 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true",
} {
if !strings.Contains(dryRunOutput.String(), want) {
t.Fatalf("dry-run output = %q, want substring %q", dryRunOutput.String(), want)
@@ -1722,6 +1670,12 @@ func TestRunForceReplacementStaysWithinRemoteBundlePaths(t *testing.T) {
testutil.AssertFakeFile(t, sshDestination, "bundle/report.md", "# Report\nSunny.\n")
testutil.AssertFakeMissing(t, sshDestination, "bundle/old.txt")
testutil.AssertFakeFile(t, sshDestination, "bundle-sibling/keep.txt", "keep")
for name, backend := range map[string]*fake.Backend{"s3": s3Destination, "ssh": sshDestination} {
catalog := readFakeCatalogStateAt(t, backend, "bundle")
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
t.Fatalf("%s catalog identity = schema %d mode %s", name, catalog.SchemaVersion, catalog.State.Mode)
}
}
}
func TestRunDryRunDoesNotWrite(t *testing.T) {
@@ -1779,7 +1733,7 @@ func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
return testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
}
func writeSharedRootLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
func writeLocalConfigWithWorkflow(t *testing.T, sourceRoot, destinationRoot, pathMapping, workflow string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
@@ -1791,35 +1745,9 @@ pipelines:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
`)
}
func writeTwoPipelineSharedRootConfig(t *testing.T, firstSourceRoot, secondSourceRoot, destinationRoot string) string {
t.Helper()
return writeConfigFile(t, `
pipelines:
- id: reports-first
source:
backend: local
path: `+firstSourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
- id: reports-second
source:
backend: local
path: `+secondSourceRoot+`
destinations:
- id: archive
backend: local
path: `+destinationRoot+`
state:
mode: shared_root
workflow: `+workflow+`
path_mapping:
mode: `+pathMapping+`
`)
}
@@ -1879,11 +1807,6 @@ func writeConfigFile(t *testing.T, body string) string {
return path
}
func writeDestinationState(t *testing.T, root, relative string, manifest bundle.Manifest) {
t.Helper()
testutil.WriteDestinationState(t, root, relative, manifest, testutil.DestinationStateOptions{})
}
func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
t.Helper()
if err := os.MkdirAll(root, 0o755); err != nil {
@@ -1898,26 +1821,93 @@ func writeJSONManifest(t *testing.T, root string, manifest bundle.Manifest) {
}
}
func readStateFile(t *testing.T, path string) state.DistributorState {
t.Helper()
return testutil.ReadDestinationState(t, path)
type testDestinationState struct {
PipelineID string
DestinationID string
SourceID string
SourceDigest string
SourceCreated time.Time
PrimaryURL string
Outputs []testStateOutput
}
func readSharedRootStateFile(t *testing.T, path string) state.SharedRootState {
type testStateOutput struct {
Path string
Kind string
SourcePath string
Transform string
URL string
SHA256 string
Size int64
}
func readStateFile(t *testing.T, path string) testDestinationState {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read shared-root state: %v", err)
t.Fatalf("read destination state: %v", err)
}
destinationState, err := state.ParseSharedRoot(data)
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse shared-root state: %v", err)
t.Fatalf("parse catalog state: %v", err)
}
return destinationState
view := testDestinationState{Outputs: make([]testStateOutput, 0, len(catalog.Outputs))}
for index, output := range catalog.Outputs {
if index == 0 {
view.PipelineID = output.PipelineID
view.DestinationID = output.DestinationID
view.SourceID = output.Source.ID
view.SourceDigest = output.Source.Digest
view.SourceCreated = output.Source.Created
if output.URL != "" {
view.PrimaryURL = output.URL
}
}
view.Outputs = append(view.Outputs, testStateOutput{
Path: output.Path,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
})
}
return view
}
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
byPath := make(map[string]state.OutputFile, len(outputs))
func readLocalCatalogState(t *testing.T, destinationRoot string) state.CatalogState {
t.Helper()
data, err := os.ReadFile(filepath.Join(destinationRoot, storage.StateFileName))
if err != nil {
t.Fatalf("read catalog state: %v", err)
}
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse catalog state: %v", err)
}
return catalog
}
func readFakeCatalogStateAt(t *testing.T, backend *fake.Backend, bundlePath string) state.CatalogState {
t.Helper()
statePath, err := storage.StatePath(bundlePath)
if err != nil {
t.Fatalf("state path: %v", err)
}
data, err := backend.ReadFile(context.Background(), statePath)
if err != nil {
t.Fatalf("read catalog state: %v", err)
}
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse catalog state: %v", err)
}
return catalog
}
func outputsByPath(outputs []testStateOutput) map[string]testStateOutput {
byPath := make(map[string]testStateOutput, len(outputs))
for _, output := range outputs {
byPath[output.Path] = output
}

View File

@@ -3,11 +3,14 @@ package cli
import (
"bytes"
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
@@ -92,9 +95,10 @@ func TestExecutePruneDryRunReportsWithoutWriting(t *testing.T) {
}
assertLocalFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
assertLocalFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
assertLocalFile(t, filepath.Join(destinationRoot, "html.txt"), "other")
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,summary.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if stderr.Len() != 0 {
@@ -159,13 +163,14 @@ func TestExecutePruneApplyDeletesManagedOutputs(t *testing.T) {
if _, err := os.Stat(filepath.Join(destinationRoot, "summary.txt")); !os.IsNotExist(err) {
t.Fatalf("summary.txt stat error = %v, want not exist", err)
}
assertLocalFile(t, filepath.Join(destinationRoot, "html.txt"), "other")
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
t.Fatalf("state file stat error = %v", err)
}
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := state.ManagedOutputPaths(destinationState); len(got) != 0 {
t.Fatalf("state outputs = %#v, want none", got)
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "html.txt" {
t.Fatalf("state outputs = %q, want html.txt", got)
}
if stderr.Len() != 0 {
t.Fatalf("stderr = %q, want empty", stderr.String())
@@ -177,13 +182,16 @@ func writePruneLocalFixture(t *testing.T) (string, string) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
writeCatalogDestinationState(t, destinationRoot, manifest, true)
for _, file := range testutil.DefaultSourceFiles() {
path := filepath.Join(destinationRoot, filepath.FromSlash(file.Path))
if err := os.WriteFile(path, []byte(file.Data), 0o600); err != nil {
t.Fatalf("write destination output: %v", err)
}
}
if err := os.WriteFile(filepath.Join(destinationRoot, "html.txt"), []byte("other"), 0o600); err != nil {
t.Fatalf("write other owner output: %v", err)
}
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
t.Fatalf("write unmanaged output: %v", err)
}
@@ -208,3 +216,71 @@ pipelines:
}
return destinationRoot, configPath
}
func writeCatalogDestinationState(t *testing.T, root string, manifest bundle.Manifest, includeOtherOwner bool) {
t.Helper()
createdAt := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
outputs := []state.CatalogOutputFile{{
Path: "report.md",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}, {
Path: "summary.txt",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}}
if includeOtherOwner {
outputs = append(outputs, state.CatalogOutputFile{
Path: "html.txt",
PipelineID: "reports",
DestinationID: "html",
Source: source,
Kind: state.OutputKindSource,
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: createdAt,
UpdatedAt: createdAt,
})
}
catalog := state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: "test",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: outputs,
}
data, err := json.MarshalIndent(catalog, "", " ")
if err != nil {
t.Fatalf("marshal catalog state: %v", err)
}
if err := os.WriteFile(filepath.Join(root, storage.StateFileName), append(data, '\n'), 0o600); err != nil {
t.Fatalf("write catalog state: %v", err)
}
}
func readLocalCatalogState(t *testing.T, path string) state.CatalogState {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read catalog state: %v", err)
}
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse catalog state: %v", err)
}
return catalog
}

View File

@@ -36,9 +36,9 @@ func TestExecuteReconcileStateAppliesByDefault(t *testing.T) {
if !strings.Contains(stdout.String(), "status=changed") {
t.Fatalf("stdout = %q, want changed status", stdout.String())
}
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
t.Fatalf("state outputs = %q, want report.md", got)
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,html.txt" {
t.Fatalf("state outputs = %q, want report.md,html.txt", got)
}
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
if stderr.Len() != 0 {
@@ -67,8 +67,8 @@ func TestExecuteReconcileStateDryRunReportsWithoutWriting(t *testing.T) {
if !strings.Contains(stdout.String(), "status=would_change") {
t.Fatalf("stdout = %q, want would_change status", stdout.String())
}
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
catalog := readLocalCatalogState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "report.md,summary.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if stderr.Len() != 0 {
@@ -173,7 +173,10 @@ func writeReconcileStateLocalFixture(t *testing.T) (string, string, string) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
writeCatalogDestinationState(t, destinationRoot, manifest, true)
if err := os.WriteFile(filepath.Join(destinationRoot, "html.txt"), []byte("other"), 0o600); err != nil {
t.Fatalf("write other owner output: %v", err)
}
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
return sourceRoot, destinationRoot, configPath
}

View File

@@ -634,8 +634,8 @@ func TestExecuteRunDryRun(t *testing.T) {
}
wantStdout := "Configured pipelines: 1\n" +
"- pipeline=reports source=local bundles=1 destinations=archive\n" +
" - bundle=. destination=archive backend=local action=publish_new outputs=report.md,summary.txt reason=\"destination state is absent\"\n" +
"Final status: ok planned=1 publish_new=1 replace_older=0 force_replace=0 skipped=0 failed=0 dry_run=true fixed_path=0\n"
" - bundle=. destination=archive backend=local action=publish_new workflow=additive outputs=report.md,summary.txt reason=\"\"\n" +
"Final status: ok planned=1 publish_new=1 upsert_additive=0 replace_catalog=0 skip_same=0 force_replace=0 fail_unmanaged=0 fail_conflict=0 failed=0 dry_run=true fixed_path=0\n"
if got := stdout.String(); got != wantStdout {
t.Fatalf("stdout = %q, want %q", got, wantStdout)
}
@@ -693,6 +693,42 @@ func TestExecuteRunJSONDryRun(t *testing.T) {
}
}
func TestExecuteRunJSONReportsSkipSame(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
var firstStdout, firstStderr bytes.Buffer
if code := Execute(context.Background(), []string{"run", "--config", configPath}, &firstStdout, &firstStderr); code != exitOK {
t.Fatalf("first exit code = %d, want %d; stderr = %q", code, exitOK, firstStderr.String())
}
var stdout, stderr bytes.Buffer
code := Execute(context.Background(), []string{"run", "--config", configPath, "--format", "json"}, &stdout, &stderr)
if code != exitOK {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
}
envelope := decodeEnvelope(t, &stdout)
result := envelopeResult(t, envelope)
actions, ok := result["actions"].([]any)
if !ok || len(actions) != 1 {
t.Fatalf("actions = %#v, want one action", result["actions"])
}
action, ok := actions[0].(map[string]any)
if !ok || action["action"] != "skip_same" {
t.Fatalf("action = %#v, want skip_same", actions[0])
}
summary, ok := result["summary"].(map[string]any)
if !ok || summary["skip_same"] != float64(1) || summary["publish_new"] != float64(0) {
t.Fatalf("summary = %#v, want skip_same counter", result["summary"])
}
if stderr.Len() != 0 {
t.Fatalf("stderr = %q, want empty", stderr.String())
}
}
func TestExecuteRunJSONDryRunReportsFixedPathMapping(t *testing.T) {
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
@@ -871,8 +907,8 @@ func TestExecuteRunJSONPartialFailure(t *testing.T) {
firstDestination := t.TempDir()
secondDestination := t.TempDir()
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged file: %v", err)
if err := os.WriteFile(filepath.Join(firstDestination, "report.md"), []byte("data"), 0o600); err != nil {
t.Fatalf("write unmanaged planned file: %v", err)
}
configPath := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configPath, []byte(`
@@ -938,8 +974,8 @@ func TestExecuteRunForceDryRunReportsWithoutWriting(t *testing.T) {
if code != exitOK {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
}
if !strings.Contains(stdout.String(), "action=force_replace") {
t.Fatalf("stdout = %q, want force_replace", stdout.String())
if !strings.Contains(stdout.String(), "action=force_replace workflow=additive") || !strings.Contains(stdout.String(), "force_replace=1") {
t.Fatalf("stdout = %q, want forced replacement", stdout.String())
}
if _, err := os.Stat(filepath.Join(destinationRoot, "unmanaged.txt")); err != nil {
t.Fatalf("unmanaged file stat error = %v", err)

View File

@@ -55,10 +55,8 @@ type Destination struct {
Transform Transform `yaml:"transform"`
PathMap PathMapping `yaml:"path_mapping"`
Links *Links `yaml:"links"`
State StatePolicy `yaml:"state"`
Reconciliation ReconciliationPolicy `yaml:"reconciliation"`
Workflow string `yaml:"workflow"`
Retention RetentionPolicy `yaml:"retention"`
Transfer TransferPolicy `yaml:"transfer"`
}
type Backend struct {
@@ -110,6 +108,7 @@ type MarkdownToHTML struct {
Enabled bool `yaml:"enabled"`
Mode string `yaml:"mode"`
Input string `yaml:"input"`
CssHref string `yaml:"css_href"`
}
type PathMapping struct {
@@ -121,14 +120,6 @@ type Links struct {
Primary string `yaml:"primary"`
}
type ReconciliationPolicy struct {
Mode string `yaml:"mode"`
}
type StatePolicy struct {
Mode string `yaml:"mode"`
}
type RetentionPolicy struct {
Prune PrunePolicy `yaml:"prune"`
}
@@ -138,10 +129,3 @@ type PrunePolicy struct {
OlderThan *Duration `yaml:"older_than"`
KeepLatest *int `yaml:"keep_latest"`
}
type TransferPolicy struct {
OnDestinationSame string `yaml:"on_destination_same"`
OnDestinationOlder string `yaml:"on_destination_older"`
OnDestinationNewer string `yaml:"on_destination_newer"`
OnConflict string `yaml:"on_conflict"`
}

View File

@@ -20,12 +20,6 @@ const (
ValidationActionFail = "fail"
)
const (
TransferActionSkip = "skip"
TransferActionReplace = "replace"
TransferActionFail = "fail"
)
const (
TransformModeSidecar = transform.MarkdownModeSidecar
TransformModeIndex = transform.MarkdownModeIndex
@@ -43,13 +37,8 @@ const (
)
const (
ReconciliationModeReplace = "replace"
ReconciliationModeMerge = "merge"
)
const (
StateModeSingleOwner = "single_owner"
StateModeSharedRoot = "shared_root"
WorkflowAdditive = "additive"
WorkflowReplacement = "replacement"
)
const DefaultS3Region = "us-east-1"
@@ -89,23 +78,8 @@ func ApplyDefaults(cfg *Config) {
if destination.Links != nil && destination.Links.Primary == "" {
destination.Links.Primary = LinkPrimaryAuto
}
if destination.State.Mode == "" {
destination.State.Mode = StateModeSingleOwner
}
if destination.Reconciliation.Mode == "" {
destination.Reconciliation.Mode = ReconciliationModeReplace
}
if destination.Transfer.OnDestinationSame == "" {
destination.Transfer.OnDestinationSame = TransferActionSkip
}
if destination.Transfer.OnDestinationOlder == "" {
destination.Transfer.OnDestinationOlder = TransferActionReplace
}
if destination.Transfer.OnDestinationNewer == "" {
destination.Transfer.OnDestinationNewer = TransferActionSkip
}
if destination.Transfer.OnConflict == "" {
destination.Transfer.OnConflict = TransferActionFail
if destination.Workflow == "" {
destination.Workflow = WorkflowAdditive
}
}
}

View File

@@ -30,14 +30,8 @@ pipelines:
if got, want := cfg.Pipelines[0].Validation.OnDigestMismatch, ValidationActionFail; got != want {
t.Fatalf("validation default = %q, want %q", got, want)
}
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
t.Fatalf("transfer default = %q, want %q", got, want)
}
if got, want := destination.Reconciliation.Mode, ReconciliationModeReplace; got != want {
t.Fatalf("reconciliation mode default = %q, want %q", got, want)
}
if got, want := destination.State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("state mode default = %q, want %q", got, want)
if got, want := destination.Workflow, WorkflowAdditive; got != want {
t.Fatalf("workflow default = %q, want %q", got, want)
}
if destination.Retention.Prune.Enabled {
t.Fatal("retention.prune.enabled default = true, want false")
@@ -173,7 +167,7 @@ pipelines:
}
}
func TestLoadFileAcceptsExplicitReconciliationModes(t *testing.T) {
func TestLoadFileAcceptsExplicitWorkflows(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: reports
@@ -184,50 +178,19 @@ pipelines:
- id: archive
backend: local
path: /archive
reconciliation:
mode: replace
workflow: additive
- id: web
backend: local
path: /web
reconciliation:
mode: merge
workflow: replacement
`)
destinations := cfg.Pipelines[0].Destinations
if got, want := destinations[0].Reconciliation.Mode, ReconciliationModeReplace; got != want {
t.Fatalf("archive reconciliation mode = %q, want %q", got, want)
if got, want := destinations[0].Workflow, WorkflowAdditive; got != want {
t.Fatalf("archive workflow = %q, want %q", got, want)
}
if got, want := destinations[1].Reconciliation.Mode, ReconciliationModeMerge; got != want {
t.Fatalf("web reconciliation mode = %q, want %q", got, want)
}
}
func TestLoadFileAcceptsExplicitStateModes(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
state:
mode: single_owner
- id: web
backend: local
path: /web
state:
mode: shared_root
`)
destinations := cfg.Pipelines[0].Destinations
if got, want := destinations[0].State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("archive state mode = %q, want %q", got, want)
}
if got, want := destinations[1].State.Mode, StateModeSharedRoot; got != want {
t.Fatalf("web state mode = %q, want %q", got, want)
if got, want := destinations[1].Workflow, WorkflowReplacement; got != want {
t.Fatalf("web workflow = %q, want %q", got, want)
}
}
@@ -852,8 +815,63 @@ pipelines:
`, "backend ftp is unsupported")
}
func TestLoadFileRejectsInvalidTransferAction(t *testing.T) {
func TestLoadFileRejectsInvalidWorkflow(t *testing.T) {
assertLoadError(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
workflow: append
`, "workflow must be additive or replacement")
}
func TestLoadFileRejectsLegacyDestinationPolicyFields(t *testing.T) {
tests := map[string]string{
"state": `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
state:
mode: single_owner
`,
"reconciliation": `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
reconciliation:
mode: replace
`,
"takeover": `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
takeover:
mode: same_pipeline
`,
"transfer": `
pipelines:
- id: reports
source:
@@ -864,8 +882,14 @@ pipelines:
backend: local
path: /archive
transfer:
on_destination_older: overwrite
`, "on_destination_older must be replace or fail")
on_destination_older: replace
`,
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
assertLoadError(t, body, "field "+name+" not found")
})
}
}
func TestLoadFileRejectsInvalidValidationAction(t *testing.T) {
@@ -934,19 +958,14 @@ pipelines:
}
func TestExampleConfigsLoad(t *testing.T) {
for _, path := range []string{
"../../examples/local-to-local.yml",
"../../examples/local-publish.yml",
"../../examples/local-html.yml",
"../../examples/local-index.yml",
"../../examples/fan-out.yml",
"../../examples/archive-and-latest.yml",
"../../examples/merge-reconciliation.yml",
"../../examples/shared-root.yml",
"../../examples/http-upload-local.yml",
"../../examples/ssh-destination.yml",
"../../examples/s3-destination.yml",
} {
paths, err := filepath.Glob("../../examples/*.yml")
if err != nil {
t.Fatalf("glob examples: %v", err)
}
if len(paths) == 0 {
t.Fatal("no example configs found")
}
for _, path := range paths {
t.Run(path, func(t *testing.T) {
if _, err := LoadFile(path); err != nil {
t.Fatalf("LoadFile(%q) error = %v", path, err)

View File

@@ -2,8 +2,10 @@ package config
import (
"fmt"
"net/url"
"regexp"
"strings"
"unicode"
"gitea.maximumdirect.net/eric/distributor/internal/link"
)
@@ -72,10 +74,8 @@ func Validate(cfg Config) error {
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
errs = validatePathMapping(errs, destinationContext+".path_mapping", destination.PathMap)
errs = validateLinks(errs, destinationContext+".links", destination.Links)
errs = validateStatePolicy(errs, destinationContext+".state", destination.State)
errs = validateReconciliationPolicy(errs, destinationContext+".reconciliation", destination.Reconciliation)
errs = validateWorkflow(errs, destinationContext+".workflow", destination.Workflow)
errs = validateRetentionPolicy(errs, destinationContext+".retention", destination.Retention)
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
}
}
@@ -281,9 +281,15 @@ func ValidatePublishTransformPolicy(publish PublishPolicy, transform Transform)
if transform.MarkdownToHTML.Input != "" && !transform.MarkdownToHTML.Enabled {
return fmt.Errorf("transform.markdown_to_html.input requires transform.markdown_to_html.enabled to be true")
}
if transform.MarkdownToHTML.CssHref != "" && !transform.MarkdownToHTML.Enabled {
return fmt.Errorf("transform.markdown_to_html.css_href requires transform.markdown_to_html.enabled to be true")
}
if transform.MarkdownToHTML.Input != "" && mode != TransformModeIndex {
return fmt.Errorf("transform.markdown_to_html.input is only valid when mode is %s", TransformModeIndex)
}
if err := validateCSSHref(transform.MarkdownToHTML.CssHref); err != nil {
return fmt.Errorf("transform.markdown_to_html.css_href %w", err)
}
if transform.MarkdownToHTML.Enabled && !publish.HTML {
return fmt.Errorf("transform.markdown_to_html.enabled requires publish.html to be true")
}
@@ -293,6 +299,49 @@ func ValidatePublishTransformPolicy(publish PublishPolicy, transform Transform)
return nil
}
func validateCSSHref(value string) error {
if value == "" {
return nil
}
for _, character := range value {
if unicode.IsControl(character) || unicode.IsSpace(character) {
return fmt.Errorf("must not contain whitespace or control characters")
}
}
if strings.ContainsAny(value, "\\<>\"'") {
return fmt.Errorf("must not contain backslashes or HTML-sensitive characters")
}
if strings.HasPrefix(value, "//") {
return fmt.Errorf("must not be scheme-relative")
}
parsed, err := url.Parse(value)
if err != nil {
return fmt.Errorf("must be a valid URL reference: %w", err)
}
if parsed.Fragment != "" {
return fmt.Errorf("must not include a fragment")
}
if parsed.Scheme != "" {
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return fmt.Errorf("scheme must be http or https")
}
if parsed.Host == "" {
return fmt.Errorf("host is required for absolute URLs")
}
if parsed.User != nil {
return fmt.Errorf("must not include userinfo")
}
return nil
}
if parsed.Host != "" {
return fmt.Errorf("must not be scheme-relative")
}
if parsed.Path == "" {
return fmt.Errorf("relative URL path is required")
}
return nil
}
func validatePathMapping(errs ValidationErrors, context string, mapping PathMapping) ValidationErrors {
if mapping.Mode != PathMappingPreserveRelative && mapping.Mode != PathMappingFixed {
errs = append(errs, context+".mode must be "+PathMappingPreserveRelative+" or "+PathMappingFixed)
@@ -317,9 +366,9 @@ func validateLinks(errs ValidationErrors, context string, links *Links) Validati
return errs
}
func validateReconciliationPolicy(errs ValidationErrors, context string, policy ReconciliationPolicy) ValidationErrors {
if policy.Mode != ReconciliationModeReplace && policy.Mode != ReconciliationModeMerge {
errs = append(errs, context+".mode must be "+ReconciliationModeReplace+" or "+ReconciliationModeMerge)
func validateWorkflow(errs ValidationErrors, context, workflow string) ValidationErrors {
if workflow != WorkflowAdditive && workflow != WorkflowReplacement {
errs = append(errs, context+" must be "+WorkflowAdditive+" or "+WorkflowReplacement)
}
return errs
}
@@ -340,26 +389,3 @@ func validateRetentionPolicy(errs ValidationErrors, context string, policy Reten
}
return errs
}
func validateStatePolicy(errs ValidationErrors, context string, policy StatePolicy) ValidationErrors {
if policy.Mode != StateModeSingleOwner && policy.Mode != StateModeSharedRoot {
errs = append(errs, context+".mode must be "+StateModeSingleOwner+" or "+StateModeSharedRoot)
}
return errs
}
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
errs = append(errs, context+".on_destination_same must be skip or fail")
}
if policy.OnDestinationOlder != TransferActionReplace && policy.OnDestinationOlder != TransferActionFail {
errs = append(errs, context+".on_destination_older must be replace or fail")
}
if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail && policy.OnDestinationNewer != TransferActionReplace {
errs = append(errs, context+".on_destination_newer must be skip, replace, or fail")
}
if policy.OnConflict != TransferActionFail && policy.OnConflict != TransferActionReplace {
errs = append(errs, context+".on_conflict must be fail or replace")
}
return errs
}

View File

@@ -51,29 +51,6 @@ func TestValidateChecksPublishTransformPolicy(t *testing.T) {
}
}
func TestValidateAcceptsForceReplacementTransferActions(t *testing.T) {
cfg := Config{Pipelines: []Pipeline{{
ID: "reports",
Source: Backend{
Backend: BackendLocal,
Path: "/source",
},
Destinations: []Destination{{
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
Transfer: TransferPolicy{
OnDestinationNewer: TransferActionReplace,
OnConflict: TransferActionReplace,
},
}},
}}}
ApplyDefaults(&cfg)
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestValidatePathMapping(t *testing.T) {
tests := []struct {
name string
@@ -108,15 +85,15 @@ func TestValidatePathMapping(t *testing.T) {
}
}
func TestValidateReconciliationPolicy(t *testing.T) {
func TestValidateWorkflow(t *testing.T) {
tests := []struct {
name string
mode string
workflow string
wantErr bool
}{
{name: "replace", mode: ReconciliationModeReplace},
{name: "merge", mode: ReconciliationModeMerge},
{name: "invalid", mode: "append", wantErr: true},
{name: "additive", workflow: WorkflowAdditive},
{name: "replacement", workflow: WorkflowReplacement},
{name: "invalid", workflow: "append", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -127,7 +104,7 @@ func TestValidateReconciliationPolicy(t *testing.T) {
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
Reconciliation: ReconciliationPolicy{Mode: tt.mode},
Workflow: tt.workflow,
}},
}}}
ApplyDefaults(&cfg)
@@ -142,18 +119,7 @@ func TestValidateReconciliationPolicy(t *testing.T) {
}
}
func TestValidateStatePolicy(t *testing.T) {
tests := []struct {
name string
mode string
wantErr bool
}{
{name: "single owner", mode: StateModeSingleOwner},
{name: "shared root", mode: StateModeSharedRoot},
{name: "invalid", mode: "shared", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
func TestValidateWorkflowReportsFieldContext(t *testing.T) {
cfg := Config{Pipelines: []Pipeline{{
ID: "reports",
Source: Backend{Backend: BackendLocal, Path: "/source"},
@@ -161,18 +127,17 @@ func TestValidateStatePolicy(t *testing.T) {
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
State: StatePolicy{Mode: tt.mode},
Workflow: "append",
}},
}}}
ApplyDefaults(&cfg)
err := Validate(cfg)
if tt.wantErr && err == nil {
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
if !tt.wantErr && err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
want := "pipelines[0].destinations[0].workflow must be additive or replacement"
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want %q", err, want)
}
}
@@ -328,6 +293,24 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
Input: "report.md",
}},
},
{
name: "html only sidecar css href allowed",
publish: PublishPolicy{HTML: true},
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
Enabled: true,
Mode: TransformModeSidecar,
CssHref: "/assets/report.css",
}},
},
{
name: "html only index css href allowed",
publish: PublishPolicy{HTML: true},
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
Enabled: true,
Mode: TransformModeIndex,
CssHref: "assets/report.css?v=20260614",
}},
},
{
name: "source and html sidecar allowed",
publish: PublishPolicy{Source: true, HTML: true},
@@ -425,6 +408,16 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
}},
wantErr: true,
},
{
name: "disabled markdown css href rejected",
publish: PublishPolicy{Source: true},
transform: Transform{MarkdownToHTML: &MarkdownToHTML{
Enabled: false,
Mode: TransformModeSidecar,
CssHref: "/assets/report.css",
}},
wantErr: true,
},
{
name: "disabled markdown wrong mode rejected",
publish: PublishPolicy{Source: true},
@@ -436,3 +429,44 @@ func publishTransformPolicyCases() []publishTransformPolicyCase {
},
}
}
func TestValidateCSSHref(t *testing.T) {
tests := []struct {
name string
value string
wantErr bool
}{
{name: "empty"},
{name: "root relative", value: "/assets/report.css"},
{name: "relative", value: "assets/report.css"},
{name: "parent relative", value: "../assets/report.css"},
{name: "query", value: "/assets/report.css?v=20260614"},
{name: "http", value: "http://example.com/report.css"},
{name: "https", value: "https://example.com/assets/report.css?v=1"},
{name: "javascript", value: "javascript:alert(1)", wantErr: true},
{name: "data", value: "data:text/css,body{}", wantErr: true},
{name: "file", value: "file:///tmp/report.css", wantErr: true},
{name: "scheme relative", value: "//example.com/report.css", wantErr: true},
{name: "userinfo", value: "https://user@example.com/report.css", wantErr: true},
{name: "fragment", value: "/assets/report.css#main", wantErr: true},
{name: "space", value: "/assets/report css", wantErr: true},
{name: "tab", value: "/assets/report\tcss", wantErr: true},
{name: "newline", value: "/assets/report\ncss", wantErr: true},
{name: "backslash", value: `assets\report.css`, wantErr: true},
{name: "less than", value: "/assets/<report>.css", wantErr: true},
{name: "double quote", value: `/assets/"report".css`, wantErr: true},
{name: "single quote", value: "/assets/'report'.css", wantErr: true},
{name: "query only", value: "?v=1", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateCSSHref(tt.value)
if tt.wantErr && err == nil {
t.Fatal("validateCSSHref() error = nil, want error")
}
if !tt.wantErr && err != nil {
t.Fatalf("validateCSSHref() error = %v", err)
}
})
}
}

View File

@@ -4,39 +4,24 @@ import (
"context"
"encoding/json"
"fmt"
"time"
"sort"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
)
func Execute(ctx context.Context, req Request, plan Plan) error {
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
switch plan.Action {
case ActionSkipSame, ActionSkipDestinationNewer:
case ActionSkipSame:
return nil
case ActionPublishNew, ActionReplaceOlder, ActionForceReplace:
if usesSharedRootState(req, plan) {
return executeSharedRoot(ctx, req, plan)
}
case ActionPublishNew, ActionUpsertAdditive, ActionReplaceCatalog, ActionForceReplace:
return executeCatalog(ctx, req, plan)
default:
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
}
}
if plan.Action == ActionReplaceOlder {
if plan.ExistingState == nil {
return fmt.Errorf("replace requires existing destination state")
}
if plan.Reconciliation.Mode == config.ReconciliationModeReplace {
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
return err
}
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
return err
}
}
}
func executeCatalog(ctx context.Context, req Request, plan Plan) error {
if plan.Action == ActionForceReplace {
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
return err
@@ -44,121 +29,26 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
return err
}
}
if usesMergeRetention(plan) {
if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil {
return err
}
}
writtenOutputs := make([]Output, 0, len(plan.Outputs))
newOutputs := make([]Output, 0, len(plan.Outputs))
cleanup := func() {
outputs := writtenOutputs
if usesMergeRetention(plan) {
outputs = newOutputs
}
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
}
for _, output := range plan.Outputs {
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
if err != nil {
cleanup()
return err
}
data := output.Data
if output.Kind == state.OutputKindSource {
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
if err != nil {
cleanup()
return err
}
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
if err != nil {
cleanup()
return err
}
}
managed := outputManagedByExistingState(output, plan.ExistingState)
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil {
cleanup()
return err
}
writtenOutputs = append(writtenOutputs, output)
if !managed {
newOutputs = append(newOutputs, output)
}
}
now := time.Now().UTC()
createdAt := now
if plan.ExistingState != nil {
createdAt = plan.ExistingState.CreatedAt
}
stateOutputs, err := stateOutputsForPlan(plan, now)
if err != nil {
cleanup()
return err
}
destinationState := state.DistributorState{
SchemaVersion: state.SchemaVersion,
DistributorVersion: req.DistributorVersion,
PipelineID: req.PipelineID,
DestinationID: req.DestinationID,
PublishedAt: now,
CreatedAt: createdAt,
UpdatedAt: now,
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
Outputs: stateOutputs,
}
if plan.PrimaryURL != "" {
destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
}
if err := state.Validate(destinationState); err != nil {
cleanup()
return err
}
data, err := json.MarshalIndent(destinationState, "", " ")
if err != nil {
cleanup()
return err
}
data = append(data, '\n')
statePath, err := storage.StatePath(req.DestinationBundlePath)
if err != nil {
cleanup()
return err
}
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil {
cleanup()
return err
}
return nil
}
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
if plan.Action == ActionForceReplace {
} else if plan.Action == ActionReplaceCatalog {
if plan.ClearDestinationRoot {
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
return err
}
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
return err
}
}
if plan.Action == ActionReplaceOlder && plan.Reconciliation.Mode == config.ReconciliationModeReplace {
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
} else if len(plan.CatalogOutputsToDelete) > 0 {
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, catalogOutputPaths(plan.CatalogOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
return err
}
}
}
writtenOutputs := make([]Output, 0, len(plan.Outputs))
newOutputs := make([]Output, 0, len(plan.Outputs))
cleanup := func() {
outputs := writtenOutputs
if plan.Reconciliation.Mode == config.ReconciliationModeMerge {
if plan.Action == ActionUpsertAdditive {
outputs = newOutputs
}
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
@@ -169,6 +59,11 @@ func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
cleanup()
return err
}
created, err := catalogWriteCreatesOutput(ctx, req.DestinationBackend, destinationPath)
if err != nil {
cleanup()
return err
}
data := output.Data
if output.Kind == state.OutputKindSource {
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
@@ -182,28 +77,22 @@ func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
return err
}
}
managed := outputManagedBySharedRootPlan(output, plan)
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil {
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: catalogOutputOverwriteAllowed(plan, output), PreferAtomic: true}); err != nil {
cleanup()
return err
}
writtenOutputs = append(writtenOutputs, output)
if !managed {
if created {
newOutputs = append(newOutputs, output)
}
}
now := time.Now().UTC()
sharedRootState, err := sharedRootStateForPlan(req, plan, now)
if err != nil {
catalogState := catalogStateForPlan(req, plan)
if err := state.ValidateCatalog(catalogState); err != nil {
cleanup()
return err
}
if err := state.ValidateSharedRoot(sharedRootState); err != nil {
cleanup()
return err
}
data, err := json.MarshalIndent(sharedRootState, "", " ")
data, err := json.MarshalIndent(catalogState, "", " ")
if err != nil {
cleanup()
return err
@@ -214,165 +103,73 @@ func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
cleanup()
return err
}
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: catalogStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
cleanup()
return err
}
return nil
}
func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error {
for _, output := range plan.Outputs {
if outputManagedByExistingState(output, plan.ExistingState) {
continue
}
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
if err != nil {
return err
}
func catalogWriteCreatesOutput(ctx context.Context, backend storage.Backend, destinationPath string) (bool, error) {
if _, err := backend.Stat(ctx, destinationPath); err == nil {
return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath))
} else if !storage.IsNotFound(err) {
return err
return false, nil
} else if storage.IsNotFound(err) {
return true, nil
} else {
return false, err
}
}
return nil
}
func usesSharedRootState(req Request, plan Plan) bool {
return normalizeState(req.State).Mode == config.StateModeSharedRoot || plan.StateMode == config.StateModeSharedRoot
}
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
if existing == nil {
func catalogOutputOverwriteAllowed(plan Plan, output Output) bool {
if plan.ClearDestinationRoot {
return false
}
_, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath)
if plan.SupersededLegacy != nil {
return true
}
if plan.ExistingCatalog == nil {
return false
}
_, ok := state.FindCatalogOutputByPath(plan.ExistingCatalog.Outputs, output.DestinationPath)
return ok
}
func outputManagedBySharedRootPlan(output Output, plan Plan) bool {
for _, existing := range currentOwnerSharedRootOutputs(plan) {
if existing.Path == output.DestinationPath {
return true
func catalogStateForPlan(req Request, plan Plan) state.CatalogState {
now := requestTime(req)
createdAt := now
if plan.ExistingCatalog != nil {
createdAt = plan.ExistingCatalog.CreatedAt
}
outputs := make([]state.CatalogOutputFile, 0, len(plan.CatalogOutputsToRetain)+len(plan.CatalogOutputsToWrite))
outputs = append(outputs, plan.CatalogOutputsToRetain...)
outputs = append(outputs, plan.CatalogOutputsToWrite...)
sort.SliceStable(outputs, func(i, j int) bool {
if outputs[i].Path != outputs[j].Path {
return outputs[i].Path < outputs[j].Path
}
return false
if outputs[i].PipelineID != outputs[j].PipelineID {
return outputs[i].PipelineID < outputs[j].PipelineID
}
func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) {
existingOutputs := []state.OutputFile(nil)
if plan.ExistingState != nil {
existingOutputs = plan.ExistingState.Outputs
}
planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now)
if !usesMergeRetention(plan) || plan.ExistingState == nil {
return planned, nil
}
return state.MergeOutputFiles(plan.ExistingState.Outputs, planned)
}
func usesMergeRetention(plan Plan) bool {
return plan.Reconciliation.Mode == config.ReconciliationModeMerge && plan.Action == ActionReplaceOlder
}
func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) {
now = now.UTC()
scope := plan.OwnerScope
if scope.PipelineID == "" && scope.DestinationID == "" {
scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
}
base := sharedRootBaseState(req, plan, now)
owner := state.OwnerRecord{
Scope: scope,
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
}
if plan.PrimaryURL != "" {
owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
}
planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now)
if plan.Action == ActionReplaceOlder && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
return state.MergeOwnerOutputs(base, scope, owner, planned)
}
return state.ReplaceOwnerOutputs(base, scope, owner, planned)
}
func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState {
if plan.Action == ActionForceReplace {
return newSharedRootState(req, now)
}
if plan.ExistingSharedRoot != nil {
base := *plan.ExistingSharedRoot
base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...)
base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...)
base.DistributorVersion = req.DistributorVersion
base.UpdatedAt = now
return base
}
if plan.ExistingState != nil {
base := newSharedRootState(req, now)
base.CreatedAt = plan.ExistingState.CreatedAt
base.UpdatedAt = now
return base
}
return newSharedRootState(req, now)
}
func newSharedRootState(req Request, now time.Time) state.SharedRootState {
return state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: req.DistributorVersion,
CreatedAt: now,
UpdatedAt: now,
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{},
Outputs: []state.SharedRootOutputFile{},
}
}
func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile {
if plan.ExistingSharedRoot != nil {
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs))
for _, output := range plan.ExistingSharedRoot.Outputs {
if output.Owner == plan.OwnerScope {
outputs = append(outputs, output)
}
}
return outputs
}
if plan.ExistingState != nil {
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs))
for _, output := range plan.ExistingState.Outputs {
outputs = append(outputs, state.SharedRootOutputFile{
Path: output.Path,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
Owner: plan.OwnerScope,
SourceID: plan.ExistingState.Source.Manifest.ID,
SourceDigest: plan.ExistingState.Source.Manifest.Digest,
SourceCreated: plan.ExistingState.Source.Manifest.Created,
CreatedAt: output.CreatedAt,
UpdatedAt: output.UpdatedAt,
return outputs[i].DestinationID < outputs[j].DestinationID
})
return state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: req.DistributorVersion,
CreatedAt: createdAt,
UpdatedAt: now,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: outputs,
}
return outputs
}
return nil
}
func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string {
func catalogStateWriteOverwrites(plan Plan) bool {
return plan.ExistingCatalog != nil || plan.SupersededLegacy != nil
}
func catalogOutputPaths(outputs []state.CatalogOutputFile) []string {
paths := make([]string, 0, len(outputs))
for _, output := range outputs {
paths = append(paths, output.Path)
}
return paths
}
func sharedRootStateWriteOverwrites(plan Plan) bool {
return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace
}

View File

@@ -2,99 +2,26 @@ package publish
import (
"context"
"fmt"
"io"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
"gitea.maximumdirect.net/eric/distributor/internal/transform"
)
func TestExecuteCleansUpAfterWriteFailure(t *testing.T) {
sourceBackend := fake.New()
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "summary.txt"}
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{})
req := Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: "",
Publish: config.PublishPolicy{Source: true},
Transfer: config.TransferPolicy{OnDestinationSame: config.TransferActionSkip, OnDestinationOlder: config.TransferActionReplace, OnDestinationNewer: config.TransferActionSkip, OnConflict: config.TransferActionFail},
DistributorVersion: "test",
}
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
err = Execute(context.Background(), req, plan)
if err == nil {
t.Fatal("Execute() error = nil, want error")
}
found, err := destinationBackend.HasAny(context.Background(), "")
if err != nil {
t.Fatalf("HasAny() error = %v", err)
}
if found {
t.Fatal("destination has content after failed execution")
}
func TestExecuteAdditiveWritesOutputsAndCatalog(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
testutil.WriteFakeFile(t, destinationBackend, "old.txt", "retained")
func TestExecuteReplaceDeletesOmittedManagedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
older.Digest = bundle.BundleDigest(older.Files)
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceOlder {
t.Fatalf("plan action = %s, want replace_older", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
destinationState := readFakeState(t, destinationBackend, "")
if got, want := len(destinationState.Outputs), 1; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
}
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
}
func TestExecuteMergeRetainsOmittedAndOverwritesManagedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
older.Digest = bundle.BundleDigest(older.Files)
existingState := testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
@@ -102,256 +29,199 @@ func TestExecuteMergeRetainsOmittedAndOverwritesManagedOutputs(t *testing.T) {
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
destinationState := readFakeState(t, destinationBackend, "")
outputs := outputsByPath(destinationState.Outputs)
if got, want := len(outputs), 2; got != want {
t.Fatalf("state output count = %d, want %d", got, want)
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
testutil.AssertFakeFile(t, destinationBackend, "old.txt", "retained")
catalog := readCatalogState(t, destinationBackend, "")
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
}
if !outputs["summary.txt"].CreatedAt.Equal(existingState.Outputs[1].CreatedAt) || !outputs["summary.txt"].UpdatedAt.Equal(existingState.Outputs[1].UpdatedAt) {
t.Fatalf("retained output timestamps = %#v, want existing %#v", outputs["summary.txt"], existingState.Outputs[1])
if len(catalog.Outputs) != 3 {
t.Fatalf("catalog outputs = %#v, want three outputs", catalog.Outputs)
}
if !outputs["report.md"].CreatedAt.Equal(existingState.Outputs[0].CreatedAt) || !outputs["report.md"].UpdatedAt.After(existingState.Outputs[0].UpdatedAt) {
t.Fatalf("updated output timestamps = %#v, want preserved created_at and newer updated_at", outputs["report.md"])
report, ok := state.FindCatalogOutputByPath(catalog.Outputs, "report.md")
if !ok {
t.Fatalf("catalog outputs = %#v, want report.md", catalog.Outputs)
}
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
if !report.CreatedAt.Equal(planCreatedAt) || !report.UpdatedAt.Equal(planUpdatedAt) {
t.Fatalf("report times = %s/%s, want created preserved and updated now", report.CreatedAt, report.UpdatedAt)
}
if report.SourcePath != "" {
t.Fatalf("source catalog output source_path = %q, want empty", report.SourcePath)
}
}
func TestExecuteMergeFailsOnUnmanagedDestinationPathCollision(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
older.Files = []bundle.ManifestFile{{
Path: "summary.txt",
SHA256: bundle.FileDigest([]byte("Summary\n")),
Size: int64(len("Summary\n")),
}}
older.Digest = bundle.BundleDigest(older.Files)
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
func TestExecuteReplacementDeletesCurrentOwnerAndPreservesOtherOwners(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
testutil.WriteFakeFile(t, destinationBackend, "stale.txt", "delete me")
testutil.WriteFakeFile(t, destinationBackend, "shared.txt", "keep me")
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
err = Execute(context.Background(), req, plan)
if err == nil {
t.Fatal("Execute() error = nil, want unmanaged path collision")
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "unmanaged")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
func TestExecuteMergeFailureCleansUpOnlyNewOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{
{Path: "report.md", Data: "# Report\nNew.\n"},
{Path: "new.md", Data: "new\n"},
{Path: "fail.md", Data: "fail\n"},
},
})
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "fail.md"}
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
older.Files = []bundle.ManifestFile{sourceBundle.Manifest.Files[0]}
older.Digest = bundle.BundleDigest(older.Files)
testutil.WriteFakeDestinationState(t, destinationBackend.Backend, "", older, testutil.DestinationStateOptions{})
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
testutil.AssertFakeMissing(t, destinationBackend, "stale.txt")
testutil.AssertFakeFile(t, destinationBackend, "shared.txt", "keep me")
catalog := readCatalogState(t, destinationBackend, "")
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "stale.txt"); ok {
t.Fatalf("catalog outputs = %#v, want stale.txt removed", catalog.Outputs)
}
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "shared.txt"); !ok {
t.Fatalf("catalog outputs = %#v, want shared.txt retained", catalog.Outputs)
}
}
func TestExecuteSupersededReplacementClearsDestinationRootOnly(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
req.DestinationBundlePath = "bundle"
legacyState := legacyStateDocument(2)
writeJSONState(t, destinationBackend, req.DestinationBundlePath, legacyState)
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
err = Execute(context.Background(), req, plan)
if err == nil {
t.Fatal("Execute() error = nil, want injected failure")
}
testutil.AssertFakeFile(t, destinationBackend.Backend, "report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend.Backend, "new.md")
testutil.AssertFakeMissing(t, destinationBackend.Backend, "fail.md")
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
func TestExecuteFixedPathSupportsReconciliationModes(t *testing.T) {
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
t.Run(mode, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "new", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.ID = "older.source"
older.Created = older.Created.Add(-time.Hour)
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unplanned.txt")
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "keep")
readCatalogState(t, destinationBackend, "bundle")
}
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
func TestExecuteSupersededAdditiveLeavesUnplannedFilesUnmanaged(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
legacyState := legacyStateDocument(2)
writeJSONState(t, destinationBackend, "", legacyState)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy report")
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "unplanned.txt", "leave me")
catalog := readCatalogState(t, destinationBackend, "")
if _, ok := state.FindCatalogOutputByPath(catalog.Outputs, "unplanned.txt"); ok {
t.Fatalf("catalog outputs = %#v, want unplanned file omitted", catalog.Outputs)
}
}
func TestExecuteForceReplaceClearsDestinationBundlePathOnly(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
req.DestinationBundlePath = "bundle"
req.Force = true
testutil.WriteFakeFile(t, destinationBackend, "bundle/report.md", "old report")
testutil.WriteFakeFile(t, destinationBackend, "bundle/unplanned.txt", "remove")
testutil.WriteFakeFile(t, destinationBackend, "bundle-child/keep.txt", "keep")
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "keep")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionForceReplace {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionForceReplace)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "bundle/summary.txt", "Summary\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unplanned.txt")
testutil.AssertFakeFile(t, destinationBackend, "bundle-child/keep.txt", "keep")
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "keep")
catalog := readCatalogState(t, destinationBackend, "bundle")
if catalog.SchemaVersion != state.CatalogSchemaVersion || catalog.State.Mode != state.StateModeCatalog {
t.Fatalf("catalog identity = schema %d mode %s", catalog.SchemaVersion, catalog.State.Mode)
}
if len(catalog.Outputs) != 2 {
t.Fatalf("catalog outputs = %#v, want planned outputs only", catalog.Outputs)
}
}
func TestExecuteForceReplaceClearsFixedDestinationRoot(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
req.DestinationBundlePath = ""
req.PathMapping = config.PathMappingFixed
req.Force = true
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old report")
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "remove")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceOlder {
t.Fatalf("plan action = %s, want replace_older", plan.Action)
if plan.Action != ActionForceReplace || storage.DisplayPath(plan.DestinationBundlePath) != "." {
t.Fatalf("plan action=%s destination=%s, want force_replace at root", plan.Action, storage.DisplayPath(plan.DestinationBundlePath))
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
destinationState := readFakeState(t, destinationBackend, "")
if got, want := destinationState.Reconciliation.Mode, mode; got != want {
t.Fatalf("state reconciliation mode = %q, want %q", got, want)
}
})
}
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nSunny.\n")
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "Summary\n")
testutil.AssertFakeMissing(t, destinationBackend, "unplanned.txt")
readCatalogState(t, destinationBackend, "")
}
func TestExecuteReconciliationModesHonorPublishPolicies(t *testing.T) {
tests := []struct {
name string
publish config.PublishPolicy
transform config.Transform
transformer TransformerResolver
wantPaths []string
}{
{
name: "source only",
publish: config.PublishPolicy{Source: true},
wantPaths: []string{"report.md"},
},
{
name: "generated only",
publish: config.PublishPolicy{HTML: true},
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
Path: "report.html",
SourcePath: "report.md",
Transform: transform.MarkdownToHTML,
Data: []byte("<h1>Report</h1>\n"),
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
Size: int64(len("<h1>Report</h1>\n")),
}}}},
wantPaths: []string{"report.html"},
},
{
name: "source and generated",
publish: config.PublishPolicy{Source: true, HTML: true},
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
Path: "report.html",
SourcePath: "report.md",
Transform: transform.MarkdownToHTML,
Data: []byte("<h1>Report</h1>\n"),
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
Size: int64(len("<h1>Report</h1>\n")),
}}}},
wantPaths: []string{"report.md", "report.html"},
},
}
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
for _, tt := range tests {
t.Run(mode+" "+tt.name, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
older := sourceBundle.Manifest
older.Created = older.Created.Add(-time.Hour)
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
req.Publish = tt.publish
req.Transform = tt.transform
req.Transformers = tt.transformer
func TestExecuteFailedWriteDoesNotWriteCatalogState(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
if err := destinationBackend.AddDirectory("report.md"); err != nil {
t.Fatalf("add conflicting directory: %v", err)
}
destinationState := readFakeState(t, destinationBackend, "")
outputs := outputsByPath(destinationState.Outputs)
for _, path := range tt.wantPaths {
if _, ok := outputs[path]; !ok {
t.Fatalf("state outputs = %#v, missing %s", destinationState.Outputs, path)
}
}
})
err = Execute(context.Background(), req, plan)
if err == nil {
t.Fatal("Execute() error = nil, want write failure")
}
if _, statErr := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(statErr) {
t.Fatalf("state stat error = %v, want missing state", statErr)
}
}
type failingBackend struct {
*fake.Backend
failPath string
}
func (b *failingBackend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
if path == b.failPath {
return storage.Entry{}, fmt.Errorf("injected write failure")
}
return b.Backend.WriteFile(ctx, path, data, opts)
}
func (b *failingBackend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
if path == b.failPath {
return storage.Entry{}, fmt.Errorf("injected write failure")
}
return b.Backend.WriteFrom(ctx, path, r, opts)
}
func testRequest(sourceBackend storage.Backend, destinationBackend storage.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
return Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: sourceBundle.RootRelativePath,
Publish: config.PublishPolicy{Source: true},
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
Transfer: config.TransferPolicy{
OnDestinationSame: config.TransferActionSkip,
OnDestinationOlder: config.TransferActionReplace,
OnDestinationNewer: config.TransferActionSkip,
OnConflict: config.TransferActionFail,
},
DistributorVersion: "test",
}
}
func readFakeState(t *testing.T, backend storage.Backend, bundlePath string) state.DistributorState {
func readCatalogState(t *testing.T, backend *fake.Backend, relative string) state.CatalogState {
t.Helper()
statePath, err := storage.StatePath(bundlePath)
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
data, err := backend.ReadFile(context.Background(), statePath)
if err != nil {
t.Fatalf("read state: %v", err)
t.Fatalf("read catalog state: %v", err)
}
destinationState, err := state.Parse(data)
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse state: %v", err)
t.Fatalf("parse catalog state: %v", err)
}
return destinationState
}
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
byPath := make(map[string]state.OutputFile, len(outputs))
for _, output := range outputs {
byPath[output.Path] = output
}
return byPath
return catalog
}

View File

@@ -1,193 +0,0 @@
package publish
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
func TestBuildPlansForcedReplacementOnlyWhenExplicit(t *testing.T) {
tests := []struct {
name string
prepare func(t *testing.T, backend *fake.Backend, source bundle.Manifest)
transfer config.TransferPolicy
wantReason string
forceAction bool
}{
{
name: "unmanaged content",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
testutil.WriteFakeFile(t, backend, "bundle/old.txt", "old")
},
transfer: defaultTransfer(),
wantReason: "fail_unmanaged",
forceAction: true,
},
{
name: "different source id",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
conflict := source
conflict.ID = "other.source"
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
},
transfer: conflictReplaceTransfer(),
wantReason: "requires --force",
forceAction: true,
},
{
name: "same created digest conflict",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
conflict := testutil.ValidManifest(testutil.BundleOptions{Files: []testutil.SourceFile{{Path: "report.md", Data: "# Different\n"}}})
testutil.WriteFakeDestinationState(t, backend, "bundle", conflict, testutil.DestinationStateOptions{})
},
transfer: conflictReplaceTransfer(),
wantReason: "requires --force",
forceAction: true,
},
{
name: "pipeline mismatch",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{PipelineID: "other-pipeline"})
},
transfer: conflictReplaceTransfer(),
wantReason: "requires --force",
forceAction: true,
},
{
name: "destination mismatch",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
testutil.WriteFakeDestinationState(t, backend, "bundle", source, testutil.DestinationStateOptions{DestinationID: "other-destination"})
},
transfer: conflictReplaceTransfer(),
wantReason: "requires --force",
forceAction: true,
},
{
name: "newer destination",
prepare: func(t *testing.T, backend *fake.Backend, source bundle.Manifest) {
t.Helper()
newer := source
newer.Created = newer.Created.AddDate(0, 0, 1)
testutil.WriteFakeDestinationState(t, backend, "bundle", newer, testutil.DestinationStateOptions{})
},
transfer: newerReplaceTransfer(),
wantReason: "requires --force",
forceAction: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
tt.prepare(t, destinationBackend, sourceBundle.Manifest)
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, tt.transfer)
_, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), tt.wantReason) {
t.Fatalf("Build() error = %v, want %q", err, tt.wantReason)
}
req.Force = true
plan, err := Build(context.Background(), req)
if tt.forceAction {
if err != nil {
t.Fatalf("Build() with force error = %v", err)
}
if plan.Action != ActionForceReplace || !plan.Force {
t.Fatalf("forced plan action = %s force=%t", plan.Action, plan.Force)
}
}
})
}
}
func TestBuildRequiresConflictPolicyForStateConflicts(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
conflict := sourceBundle.Manifest
conflict.ID = "other.source"
testutil.WriteFakeDestinationState(t, destinationBackend, "bundle", conflict, testutil.DestinationStateOptions{})
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
req.Force = true
_, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "destination source id differs") {
t.Fatalf("Build() error = %v, want conservative conflict", err)
}
}
func TestExecuteForcedReplacementDeletesOnlyBundlePath(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{})
destinationBackend := fake.New()
testutil.WriteFakeFile(t, destinationBackend, "bundle/old.txt", "old")
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
testutil.WriteFakeFile(t, destinationBackend, "outside.txt", "outside")
req := forceRequest(sourceBackend, destinationBackend, sourceBundle, defaultTransfer())
req.Force = true
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionForceReplace {
t.Fatalf("plan action = %s, want force_replace", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nSunny.\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.txt")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
testutil.AssertFakeFile(t, destinationBackend, "outside.txt", "outside")
}
func forceRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, transfer config.TransferPolicy) Request {
return Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: sourceBundle.RootRelativePath,
Publish: config.PublishPolicy{Source: true},
Transfer: transfer,
DistributorVersion: "test",
}
}
func defaultTransfer() config.TransferPolicy {
return config.TransferPolicy{
OnDestinationSame: config.TransferActionSkip,
OnDestinationOlder: config.TransferActionReplace,
OnDestinationNewer: config.TransferActionSkip,
OnConflict: config.TransferActionFail,
}
}
func conflictReplaceTransfer() config.TransferPolicy {
transfer := defaultTransfer()
transfer.OnConflict = config.TransferActionReplace
return transfer
}
func newerReplaceTransfer() config.TransferPolicy {
transfer := defaultTransfer()
transfer.OnDestinationNewer = config.TransferActionReplace
return transfer
}

View File

@@ -29,6 +29,7 @@ func PlanOutputs(ctx context.Context, req Request) ([]Output, error) {
Markdown: transform.MarkdownOptions{
Mode: req.Transform.MarkdownToHTML.Mode,
Input: req.Transform.MarkdownToHTML.Input,
CssHref: req.Transform.MarkdownToHTML.CssHref,
},
})
if err != nil {
@@ -97,50 +98,10 @@ func rejectOutputCollisions(outputs []Output) error {
return nil
}
func (o Output) StateOutputFile() state.OutputFile {
return state.OutputFile{
Path: o.DestinationPath,
Kind: o.Kind,
SourcePath: o.SourcePath,
Transform: o.Transform,
URL: o.URL,
SHA256: o.SHA256,
Size: o.Size,
}
}
func (o Output) StateOutputProjection() state.OutputProjection {
return state.OutputProjection{
Path: o.DestinationPath,
Kind: o.Kind,
SourcePath: o.SourcePath,
Transform: o.Transform,
URL: o.URL,
SHA256: o.SHA256,
Size: o.Size,
}
}
func (o Output) ManagedPath() string {
return o.DestinationPath
}
func StateOutputFiles(outputs []Output) []state.OutputFile {
files := make([]state.OutputFile, 0, len(outputs))
for _, output := range outputs {
files = append(files, output.StateOutputFile())
}
return files
}
func StateOutputProjections(outputs []Output) []state.OutputProjection {
projections := make([]state.OutputProjection, 0, len(outputs))
for _, output := range outputs {
projections = append(projections, output.StateOutputProjection())
}
return projections
}
func ManagedOutputPaths(outputs []Output) []string {
paths := make([]string, 0, len(outputs))
for _, output := range outputs {

View File

@@ -12,46 +12,12 @@ import (
"gitea.maximumdirect.net/eric/distributor/internal/transform"
)
func TestOutputStateProjection(t *testing.T) {
sourceOutput := Output{
SourcePath: "report.md",
DestinationPath: "report.md",
Kind: state.OutputKindSource,
URL: "https://reports.example.com/report.md",
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Size: 123,
}
sourceState := sourceOutput.StateOutputFile()
if sourceState.Path != "report.md" || sourceState.Kind != state.OutputKindSource || sourceState.SourcePath != "report.md" || sourceState.URL != sourceOutput.URL || sourceState.SHA256 != sourceOutput.SHA256 || sourceState.Size != sourceOutput.Size {
t.Fatalf("source state output = %#v", sourceState)
}
generatedOutput := Output{
SourcePath: "report.md",
DestinationPath: "report.html",
Kind: state.OutputKindGenerated,
Transform: transform.MarkdownToHTML,
URL: "https://reports.example.com/report.html",
SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
Size: 456,
}
generatedState := generatedOutput.StateOutputFile()
if generatedState.Path != "report.html" || generatedState.Kind != state.OutputKindGenerated || generatedState.SourcePath != "report.md" || generatedState.Transform != transform.MarkdownToHTML || generatedState.URL != generatedOutput.URL || generatedState.SHA256 != generatedOutput.SHA256 || generatedState.Size != generatedOutput.Size {
t.Fatalf("generated state output = %#v", generatedState)
}
}
func TestOutputSliceProjections(t *testing.T) {
func TestManagedOutputPaths(t *testing.T) {
outputs := []Output{
{SourcePath: "report.md", DestinationPath: "report.md", Kind: state.OutputKindSource, SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Size: 1},
{SourcePath: "report.md", DestinationPath: "report.html", Kind: state.OutputKindGenerated, Transform: transform.MarkdownToHTML, URL: "https://reports.example.com/report.html", SHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Size: 2},
}
stateOutputs := StateOutputFiles(outputs)
if len(stateOutputs) != 2 || stateOutputs[1].Path != "report.html" || stateOutputs[1].Transform != transform.MarkdownToHTML || stateOutputs[1].URL != outputs[1].URL {
t.Fatalf("state outputs = %#v", stateOutputs)
}
paths := ManagedOutputPaths(outputs)
if len(paths) != 2 || paths[0] != "report.md" || paths[1] != "report.html" {
t.Fatalf("managed paths = %#v", paths)
@@ -167,6 +133,7 @@ func TestPlanOutputsPassesMarkdownOptions(t *testing.T) {
Enabled: true,
Mode: config.TransformModeIndex,
Input: "report.md",
CssHref: "/assets/report.css",
}},
Transformers: testResolver{transform.MarkdownToHTML: transformer},
})
@@ -174,8 +141,8 @@ func TestPlanOutputsPassesMarkdownOptions(t *testing.T) {
if err != nil {
t.Fatalf("PlanOutputs() error = %v", err)
}
if transformer.request.Markdown.Mode != config.TransformModeIndex || transformer.request.Markdown.Input != "report.md" {
t.Fatalf("markdown options = %#v, want index/report.md", transformer.request.Markdown)
if transformer.request.Markdown.Mode != config.TransformModeIndex || transformer.request.Markdown.Input != "report.md" || transformer.request.Markdown.CssHref != "/assets/report.css" {
t.Fatalf("markdown options = %#v, want index/report.md with css href", transformer.request.Markdown)
}
}
@@ -193,12 +160,6 @@ func TestBuildRejectsHTMLWithoutTransform(t *testing.T) {
DestinationBundlePath: "",
SourceBundle: sourceBundle,
Publish: config.PublishPolicy{HTML: true},
Transfer: config.TransferPolicy{
OnDestinationSame: config.TransferActionSkip,
OnDestinationOlder: config.TransferActionReplace,
OnDestinationNewer: config.TransferActionSkip,
OnConflict: config.TransferActionFail,
},
})
if err == nil {
t.Fatal("Build() error = nil, want missing transform error")

View File

@@ -3,6 +3,7 @@ package publish
import (
"context"
"fmt"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
@@ -15,12 +16,12 @@ type Action string
const (
ActionPublishNew Action = "publish_new"
ActionReplaceOlder Action = "replace_older"
ActionSkipSame Action = "skip_same"
ActionSkipDestinationNewer Action = "skip_destination_newer"
ActionFailConflict Action = "fail_conflict"
ActionFailUnmanaged Action = "fail_unmanaged"
ActionForceReplace Action = "force_replace"
ActionUpsertAdditive Action = "upsert_additive"
ActionReplaceCatalog Action = "replace_catalog"
)
type Request struct {
@@ -34,18 +35,28 @@ type Request struct {
Publish config.PublishPolicy
Transform config.Transform
Links *config.Links
State config.StatePolicy
Reconciliation config.ReconciliationPolicy
Workflow string
Transformers TransformerResolver
Transfer config.TransferPolicy
DistributorVersion string
Force bool
Now time.Time
}
type TransformerResolver interface {
Get(name string) (transform.Transformer, bool)
}
type Output struct {
SourcePath string
DestinationPath string
Kind string
Transform string
URL string
Data []byte
SHA256 string
Size int64
}
type Plan struct {
PipelineID string
DestinationID string
@@ -57,27 +68,24 @@ type Plan struct {
Reason string
Force bool
PrimaryURL string
StateMode string
Workflow string
OwnerScope state.OwnerScope
Reconciliation config.ReconciliationPolicy
Outputs []Output
ExistingState *state.DistributorState
ExistingSharedRoot *state.SharedRootState
OtherOwnerOutputs []state.SharedRootOutputFile
RetainedOwnerOutputs []state.SharedRootOutputFile
OwnerOutputsToDelete []state.SharedRootOutputFile
OwnerOutputsToWrite []Output
ExistingCatalog *state.CatalogState
SupersededLegacy *state.SupersededLegacyState
CatalogOutputsToWrite []state.CatalogOutputFile
CatalogOutputsToRetain []state.CatalogOutputFile
CatalogOutputsToDelete []state.CatalogOutputFile
ClearDestinationRoot bool
}
type Output struct {
SourcePath string
DestinationPath string
Kind string
Transform string
URL string
Data []byte
SHA256 string
Size int64
type catalogPlanDetails struct {
Action Action
Reason string
CatalogOutputsToWrite []state.CatalogOutputFile
CatalogOutputsToRetain []state.CatalogOutputFile
CatalogOutputsToDelete []state.CatalogOutputFile
ClearDestinationRoot bool
}
func Build(ctx context.Context, req Request) (Plan, error) {
@@ -96,10 +104,9 @@ func Build(ctx context.Context, req Request) (Plan, error) {
if err != nil {
return Plan{}, err
}
comparison := compareDestination(req, status)
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
reconciliation := normalizeReconciliation(req.Reconciliation)
stateMode := normalizeState(req.State).Mode
workflow := normalizeWorkflow(req.Workflow)
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
now := requestTime(req)
plan := Plan{
PipelineID: req.PipelineID,
DestinationID: req.DestinationID,
@@ -107,32 +114,53 @@ func Build(ctx context.Context, req Request) (Plan, error) {
BundlePath: req.SourceBundle.RootRelativePath,
DestinationBundlePath: req.DestinationBundlePath,
PathMapping: req.PathMapping,
Action: action,
Reason: reason,
Force: action == ActionForceReplace,
PrimaryURL: primaryURL,
StateMode: stateMode,
OwnerScope: state.CurrentOwnerScope(req.PipelineID, req.DestinationID),
Reconciliation: reconciliation,
Workflow: workflow,
OwnerScope: scope,
Outputs: outputs,
ExistingState: status.State,
ExistingSharedRoot: status.SharedRoot,
ExistingCatalog: status.Catalog,
SupersededLegacy: status.SupersededLegacy,
}
if status.StateErr != nil {
plan.Reason = status.StateErr.Error()
if req.Force {
plan.Action = ActionForceReplace
plan.Force = true
plan.ClearDestinationRoot = true
plan.CatalogOutputsToWrite = catalogOutputsForPlan(req, outputs, nil, scope, now)
return plan, nil
}
plan.Action = ActionFailConflict
return plan, fmt.Errorf("%s: %s", plan.Action, plan.Reason)
}
var details catalogPlanDetails
switch {
case status.Catalog != nil:
details, err = planExistingCatalog(ctx, req, *status.Catalog, outputs, workflow, scope, now)
case status.SupersededLegacy != nil:
details = planSupersededLegacy(req, outputs, workflow, scope, now)
default:
details, err = planWithoutCatalog(ctx, req, outputs, workflow, scope, now, status.HasContents)
}
plan.Action = details.Action
plan.Reason = details.Reason
plan.CatalogOutputsToWrite = details.CatalogOutputsToWrite
plan.CatalogOutputsToRetain = details.CatalogOutputsToRetain
plan.CatalogOutputsToDelete = details.CatalogOutputsToDelete
plan.ClearDestinationRoot = details.ClearDestinationRoot
if err != nil && req.Force && forceCanReplace(details.Action) {
plan.Action = ActionForceReplace
plan.Force = true
plan.ClearDestinationRoot = true
plan.CatalogOutputsToWrite = catalogOutputsForPlan(req, outputs, nil, scope, now)
plan.CatalogOutputsToRetain = nil
plan.CatalogOutputsToDelete = nil
return plan, nil
}
if stateMode == config.StateModeSharedRoot {
sharedDetails, err := planSharedRootOwner(ctx, req, status, action, reconciliation, outputs)
plan.OtherOwnerOutputs = sharedDetails.OtherOwnerOutputs
plan.RetainedOwnerOutputs = sharedDetails.RetainedOwnerOutputs
plan.OwnerOutputsToDelete = sharedDetails.OwnerOutputsToDelete
plan.OwnerOutputsToWrite = sharedDetails.OwnerOutputsToWrite
if err != nil {
plan.Action = sharedDetails.Action
plan.Reason = sharedDetails.Reason
return plan, err
}
}
if action == ActionFailConflict || action == ActionFailUnmanaged {
return plan, fmt.Errorf("%s: %s", action, reason)
}
return plan, nil
}
@@ -152,140 +180,181 @@ func validateRequest(req Request) error {
if err := config.ValidatePublishTransformPolicy(req.Publish, req.Transform); err != nil {
return fmt.Errorf("publish/transform policy: %w", err)
}
switch normalizeReconciliation(req.Reconciliation).Mode {
case config.ReconciliationModeReplace, config.ReconciliationModeMerge:
switch normalizeWorkflow(req.Workflow) {
case config.WorkflowAdditive, config.WorkflowReplacement:
default:
return fmt.Errorf("reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
}
switch normalizeState(req.State).Mode {
case config.StateModeSingleOwner, config.StateModeSharedRoot:
default:
return fmt.Errorf("state.mode must be %s or %s", config.StateModeSingleOwner, config.StateModeSharedRoot)
return fmt.Errorf("destination.workflow must be %s or %s", config.WorkflowAdditive, config.WorkflowReplacement)
}
return nil
}
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
if policy.Mode == "" {
policy.Mode = config.ReconciliationModeReplace
func normalizeWorkflow(workflow string) string {
if workflow == "" {
return config.WorkflowAdditive
}
return policy
return workflow
}
func normalizeState(policy config.StatePolicy) config.StatePolicy {
if policy.Mode == "" {
policy.Mode = config.StateModeSingleOwner
func requestTime(req Request) time.Time {
if req.Now.IsZero() {
return time.Now().UTC()
}
return policy
return req.Now.UTC()
}
func compareDestination(req Request, status state.DestinationStatus) state.Comparison {
if normalizeState(req.State).Mode == config.StateModeSharedRoot {
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
comparison := state.CompareSharedRootOwner(req.SourceBundle.Manifest, scope, status)
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict {
return comparison
func planExistingCatalog(ctx context.Context, req Request, catalog state.CatalogState, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) (catalogPlanDetails, error) {
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, catalog.Outputs, outputs); err != nil {
return catalogPlanDetails{
Action: ActionFailUnmanaged,
Reason: err.Error(),
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
}
destinationManifest, ok := sharedRootComparisonManifest(status, scope)
if !ok {
return comparison
planned := outputPathSet(outputs)
details := catalogPlanDetails{
Action: actionForWorkflow(workflow),
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, catalog.Outputs, scope, now),
}
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
}
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"}
}
return comparison
}
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict || status.State == nil {
return comparison
}
destinationManifest := status.State.Source.Manifest
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
}
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"}
}
return comparison
}
func sharedRootComparisonManifest(status state.DestinationStatus, scope state.OwnerScope) (bundle.Manifest, bool) {
if status.SharedRoot != nil {
return status.SharedRoot.SourceManifest(scope)
}
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
return status.State.Source.Manifest, true
}
return bundle.Manifest{}, false
}
type sharedRootPlanDetails struct {
Action Action
Reason string
OtherOwnerOutputs []state.SharedRootOutputFile
RetainedOwnerOutputs []state.SharedRootOutputFile
OwnerOutputsToDelete []state.SharedRootOutputFile
OwnerOutputsToWrite []Output
}
func planSharedRootOwner(ctx context.Context, req Request, status state.DestinationStatus, action Action, reconciliation config.ReconciliationPolicy, outputs []Output) (sharedRootPlanDetails, error) {
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
details := sharedRootPlanDetails{Action: action}
if !isWriteAction(action) {
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
return details, nil
}
plannedPaths := outputPaths(outputs)
if action == ActionForceReplace {
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
return details, nil
}
if conflict, ok := sharedRootPathOwnershipConflict(status, scope, plannedPaths); ok {
reason := fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
details.Action = ActionFailConflict
details.Reason = reason
return details, fmt.Errorf("%s: %s", ActionFailConflict, reason)
}
if err := rejectSharedRootUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, status, scope, plannedPaths); err != nil {
details.Action = ActionFailUnmanaged
details.Reason = err.Error()
return details, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
}
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
ownerOutputs := currentOwnerOutputs(status, scope)
planned := make(map[string]struct{}, len(plannedPaths))
for _, path := range plannedPaths {
planned[path] = struct{}{}
}
for _, output := range ownerOutputs {
allPlannedOutputsMatch := catalogContainsMatchingOutputs(req, catalog.Outputs, outputs, scope)
for _, output := range catalog.Outputs {
if _, exists := planned[output.Path]; exists {
continue
}
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeReplace {
details.OwnerOutputsToDelete = append(details.OwnerOutputsToDelete, output)
if workflow == config.WorkflowReplacement && output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
details.CatalogOutputsToDelete = append(details.CatalogOutputsToDelete, output)
continue
}
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeMerge {
details.RetainedOwnerOutputs = append(details.RetainedOwnerOutputs, output)
details.CatalogOutputsToRetain = append(details.CatalogOutputsToRetain, output)
}
if allPlannedOutputsMatch && (workflow == config.WorkflowAdditive || len(details.CatalogOutputsToDelete) == 0) {
details.Action = ActionSkipSame
details.CatalogOutputsToWrite = nil
details.CatalogOutputsToDelete = nil
}
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
return details, nil
}
func isWriteAction(action Action) bool {
switch action {
case ActionPublishNew, ActionReplaceOlder, ActionForceReplace:
return true
default:
func catalogContainsMatchingOutputs(req Request, existing []state.CatalogOutputFile, outputs []Output, scope state.OwnerScope) bool {
for _, output := range outputs {
catalogOutput, ok := state.FindCatalogOutputByPath(existing, output.DestinationPath)
if !ok || !catalogOutputMatchesPlan(req, catalogOutput, output, scope) {
return false
}
}
return true
}
func catalogOutputMatchesPlan(req Request, catalogOutput state.CatalogOutputFile, output Output, scope state.OwnerScope) bool {
if catalogOutput.PipelineID != scope.PipelineID ||
catalogOutput.DestinationID != scope.DestinationID ||
catalogOutput.Source.ID != req.SourceBundle.Manifest.ID ||
catalogOutput.Source.Digest != req.SourceBundle.Manifest.Digest ||
!catalogOutput.Source.Created.Equal(req.SourceBundle.Manifest.Created) ||
catalogOutput.Path != output.DestinationPath ||
catalogOutput.Kind != output.Kind ||
catalogOutput.URL != output.URL ||
catalogOutput.SHA256 != output.SHA256 ||
catalogOutput.Size != output.Size {
return false
}
if output.Kind == state.OutputKindGenerated {
return catalogOutput.SourcePath == output.SourcePath && catalogOutput.Transform == output.Transform
}
return catalogOutput.SourcePath == "" && catalogOutput.Transform == ""
}
func planSupersededLegacy(req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time) catalogPlanDetails {
details := catalogPlanDetails{
Action: actionForWorkflow(workflow),
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, nil, scope, now),
}
if workflow == config.WorkflowReplacement {
details.ClearDestinationRoot = true
}
return details
}
func planWithoutCatalog(ctx context.Context, req Request, outputs []Output, workflow string, scope state.OwnerScope, now time.Time, hasContents bool) (catalogPlanDetails, error) {
if hasContents {
err := fmt.Errorf("destination has content but no distributor state")
return catalogPlanDetails{
Action: ActionFailUnmanaged,
Reason: err.Error(),
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
}
if err := rejectCatalogUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, nil, outputs); err != nil {
return catalogPlanDetails{
Action: ActionFailUnmanaged,
Reason: err.Error(),
}, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
}
return catalogPlanDetails{
Action: ActionPublishNew,
CatalogOutputsToWrite: catalogOutputsForPlan(req, outputs, nil, scope, now),
}, nil
}
func forceCanReplace(action Action) bool {
return action == ActionFailUnmanaged || action == ActionFailConflict
}
func actionForWorkflow(workflow string) Action {
if workflow == config.WorkflowReplacement {
return ActionReplaceCatalog
}
return ActionUpsertAdditive
}
func catalogOutputsForPlan(req Request, outputs []Output, existing []state.CatalogOutputFile, scope state.OwnerScope, now time.Time) []state.CatalogOutputFile {
files := make([]state.CatalogOutputFile, 0, len(outputs))
source := state.CatalogSourceIdentity{
ID: req.SourceBundle.Manifest.ID,
Digest: req.SourceBundle.Manifest.Digest,
Created: req.SourceBundle.Manifest.Created,
}
for _, output := range outputs {
createdAt := now
if existingOutput, ok := state.FindCatalogOutputByPath(existing, output.DestinationPath); ok {
createdAt = existingOutput.CreatedAt
}
file := state.CatalogOutputFile{
Path: output.DestinationPath,
PipelineID: scope.PipelineID,
DestinationID: scope.DestinationID,
Source: source,
Kind: output.Kind,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
CreatedAt: createdAt,
UpdatedAt: now,
}
if output.Kind == state.OutputKindGenerated {
file.SourcePath = output.SourcePath
file.Transform = output.Transform
}
files = append(files, file)
}
return files
}
func rejectCatalogUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, existing []state.CatalogOutputFile, outputs []Output) error {
managed := catalogOutputPathSet(existing)
for _, output := range outputs {
if _, exists := managed[output.DestinationPath]; exists {
continue
}
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
if err != nil {
return err
}
if _, err := backend.Stat(ctx, destinationPath); err == nil {
return fmt.Errorf("destination output path %s exists but is not managed by catalog state", storage.DisplayPath(output.DestinationPath))
} else if !storage.IsNotFound(err) {
return err
}
}
return nil
}
func outputPaths(outputs []Output) []string {
paths := make([]string, 0, len(outputs))
@@ -295,131 +364,18 @@ func outputPaths(outputs []Output) []string {
return paths
}
func sharedRootPathOwnershipConflict(status state.DestinationStatus, scope state.OwnerScope, paths []string) (state.PathOwnershipConflict, bool) {
if status.SharedRoot != nil {
return status.SharedRoot.PathOwnershipConflict(scope, paths)
func outputPathSet(outputs []Output) map[string]struct{} {
paths := make(map[string]struct{}, len(outputs))
for _, output := range outputs {
paths[output.DestinationPath] = struct{}{}
}
return state.PathOwnershipConflict{}, false
return paths
}
func rejectSharedRootUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, status state.DestinationStatus, scope state.OwnerScope, paths []string) error {
for _, path := range paths {
if pathManagedBySharedRootStatus(status, scope, path) {
continue
}
destinationPath, err := storage.Join(bundlePath, path)
if err != nil {
return err
}
if _, err := backend.Stat(ctx, destinationPath); err == nil {
return fmt.Errorf("destination output path %s exists but is not managed by destination state", storage.DisplayPath(path))
} else if !storage.IsNotFound(err) {
return err
}
}
return nil
}
func pathManagedBySharedRootStatus(status state.DestinationStatus, scope state.OwnerScope, path string) bool {
if status.SharedRoot != nil {
_, exists := status.SharedRoot.OutputOwner(path)
return exists
}
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
_, exists := state.FindOutputByPath(status.State.Outputs, path)
return exists
}
return false
}
func otherOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
if status.SharedRoot == nil {
return nil
}
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
for _, output := range status.SharedRoot.Outputs {
if output.Owner != scope {
outputs = append(outputs, output)
}
}
return outputs
}
func currentOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
if status.SharedRoot != nil {
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
for _, output := range status.SharedRoot.Outputs {
if output.Owner == scope {
outputs = append(outputs, output)
}
}
return outputs
}
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
outputs := make([]state.SharedRootOutputFile, 0, len(status.State.Outputs))
for _, output := range status.State.Outputs {
outputs = append(outputs, state.SharedRootOutputFile{
Path: output.Path,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
Owner: scope,
SourceID: status.State.Source.Manifest.ID,
SourceDigest: status.State.Source.Manifest.Digest,
SourceCreated: status.State.Source.Manifest.Created,
CreatedAt: output.CreatedAt,
UpdatedAt: output.UpdatedAt,
})
}
return outputs
}
return nil
}
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
switch comparison.Outcome {
case state.OutcomeDestinationAbsent:
return ActionPublishNew, comparison.Reason
case state.OutcomeDestinationUnmanaged:
if force {
return ActionForceReplace, "forced replacement of unmanaged destination content"
}
return ActionFailUnmanaged, comparison.Reason
case state.OutcomeInvalidState:
return ActionFailConflict, comparison.Reason
case state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
if transfer.OnConflict == config.TransferActionReplace {
if force {
return ActionForceReplace, "forced replacement of conflicting destination state: " + comparison.Reason
}
return ActionFailConflict, "destination conflict replacement requires --force"
}
return ActionFailConflict, comparison.Reason
case state.OutcomeSameSource:
if transfer.OnDestinationSame == config.TransferActionFail {
return ActionFailConflict, "destination matches source and transfer policy requires failure"
}
return ActionSkipSame, comparison.Reason
case state.OutcomeDestinationOlder:
if transfer.OnDestinationOlder == config.TransferActionFail {
return ActionFailConflict, "destination is older and transfer policy requires failure"
}
return ActionReplaceOlder, comparison.Reason
case state.OutcomeDestinationNewer:
if transfer.OnDestinationNewer == config.TransferActionReplace {
if force {
return ActionForceReplace, "forced replacement of newer destination state"
}
return ActionFailConflict, "destination is newer and replacement requires --force"
}
if transfer.OnDestinationNewer == config.TransferActionFail {
return ActionFailConflict, "destination is newer and transfer policy requires failure"
}
return ActionSkipDestinationNewer, comparison.Reason
default:
return ActionFailConflict, "unsupported comparison outcome"
func catalogOutputPathSet(outputs []state.CatalogOutputFile) map[string]struct{} {
paths := make(map[string]struct{}, len(outputs))
for _, output := range outputs {
paths[output.Path] = struct{}{}
}
return paths
}

View File

@@ -0,0 +1,624 @@
package publish
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
"gitea.maximumdirect.net/eric/distributor/internal/transform"
)
var (
planCreatedAt = time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
planUpdatedAt = time.Date(2026, 6, 1, 9, 30, 0, 0, time.UTC)
)
func TestBuildAdditivePublishesCatalogOutputs(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionPublishNew || plan.Workflow != config.WorkflowAdditive {
t.Fatalf("plan action=%s workflow=%s, want publish_new additive", plan.Action, plan.Workflow)
}
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
}
for _, output := range plan.CatalogOutputsToWrite {
if output.PipelineID != "reports" || output.DestinationID != "archive" {
t.Fatalf("catalog output owner = %s/%s", output.PipelineID, output.DestinationID)
}
if output.Source.ID != req.SourceBundle.Manifest.ID || output.Source.Digest != req.SourceBundle.Manifest.Digest || !output.Source.Created.Equal(req.SourceBundle.Manifest.Created) {
t.Fatalf("catalog output source = %#v", output.Source)
}
if output.Kind == state.OutputKindSource && output.SourcePath != "" {
t.Fatalf("source catalog output source_path = %q, want empty", output.SourcePath)
}
if !output.CreatedAt.Equal(planUpdatedAt) || !output.UpdatedAt.Equal(planUpdatedAt) {
t.Fatalf("catalog output times = %s/%s", output.CreatedAt, output.UpdatedAt)
}
}
if _, err := destinationBackend.Stat(context.Background(), storage.StateFileName); !storage.IsNotFound(err) {
t.Fatalf("destination state stat error = %v, want missing", err)
}
}
func TestBuildAdditiveOverwritesManagedAndRetainsOthers(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "old")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionUpsertAdditive {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionUpsertAdditive)
}
if len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("delete outputs = %#v, want none", plan.CatalogOutputsToDelete)
}
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "old.txt" {
t.Fatalf("retained outputs = %#v, want old.txt", plan.CatalogOutputsToRetain)
}
written, ok := state.FindCatalogOutputByPath(plan.CatalogOutputsToWrite, "report.md")
if !ok {
t.Fatalf("written outputs = %#v, want report.md", plan.CatalogOutputsToWrite)
}
if !written.CreatedAt.Equal(planCreatedAt) || !written.UpdatedAt.Equal(planUpdatedAt) {
t.Fatalf("report.md times = %s/%s, want created preserved and updated now", written.CreatedAt, written.UpdatedAt)
}
}
func TestBuildReplacementDeletesCurrentOwnerAndRetainsOtherOwners(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "archive", "report.md", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt),
catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceCatalog {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceCatalog)
}
if len(plan.CatalogOutputsToDelete) != 1 || plan.CatalogOutputsToDelete[0].Path != "stale.txt" {
t.Fatalf("delete outputs = %#v, want stale.txt", plan.CatalogOutputsToDelete)
}
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "shared.txt" {
t.Fatalf("retained outputs = %#v, want shared.txt", plan.CatalogOutputsToRetain)
}
}
func TestBuildAdditiveSkipsMatchingCatalogOutputs(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
existing := matchingCatalogForRequest(t, req)
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "web", "old.txt", state.OutputKindSource, planCreatedAt))
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionSkipSame {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionSkipSame)
}
if len(plan.CatalogOutputsToWrite) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d delete=%d, want no writes or deletes", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToDelete))
}
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "old.txt" {
t.Fatalf("retained outputs = %#v, want old.txt", plan.CatalogOutputsToRetain)
}
}
func TestBuildReplacementSkipsMatchingCatalogOutputs(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := matchingCatalogForRequest(t, req)
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "web", "shared.txt", state.OutputKindSource, planCreatedAt))
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionSkipSame {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionSkipSame)
}
if len(plan.CatalogOutputsToWrite) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d delete=%d, want no writes or deletes", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToDelete))
}
if len(plan.CatalogOutputsToRetain) != 1 || plan.CatalogOutputsToRetain[0].Path != "shared.txt" {
t.Fatalf("retained outputs = %#v, want shared.txt", plan.CatalogOutputsToRetain)
}
}
func TestBuildReplacementDoesNotSkipWhenCurrentOwnerOutputWouldBeDeleted(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := matchingCatalogForRequest(t, req)
existing.Outputs = append(existing.Outputs, catalogOutput(req, "reports", "archive", "stale.txt", state.OutputKindSource, planCreatedAt))
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceCatalog {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionReplaceCatalog)
}
if len(plan.CatalogOutputsToDelete) != 1 || plan.CatalogOutputsToDelete[0].Path != "stale.txt" {
t.Fatalf("delete outputs = %#v, want stale.txt", plan.CatalogOutputsToDelete)
}
}
func TestBuildDoesNotSkipWhenCatalogMetadataDiffers(t *testing.T) {
tests := []struct {
name string
configure func(*Request)
mutate func(*state.CatalogState)
}{
{
name: "pipeline id",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].PipelineID = "other"
},
},
{
name: "destination id",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].DestinationID = "web"
},
},
{
name: "source id",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Source.ID = "reports.other"
},
},
{
name: "source digest",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Source.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
},
},
{
name: "source created",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Source.Created = catalog.Outputs[0].Source.Created.Add(time.Second)
},
},
{
name: "output kind",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Kind = state.OutputKindGenerated
catalog.Outputs[0].SourcePath = "report.md"
catalog.Outputs[0].Transform = transform.MarkdownToHTML
},
},
{
name: "output digest",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].SHA256 = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
},
},
{
name: "output size",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Size++
},
},
{
name: "url",
configure: func(req *Request) {
req.Links = &config.Links{BaseURL: "https://reports.example.com/archive", Primary: config.LinkPrimaryAuto}
},
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].URL = "https://reports.example.com/archive/old-report.md"
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
if tt.configure != nil {
tt.configure(&req)
}
existing := matchingCatalogForRequest(t, req)
tt.mutate(&existing)
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action == ActionSkipSame {
t.Fatalf("plan action = %s, want write action after metadata change", plan.Action)
}
})
}
}
func TestBuildDoesNotSkipWhenGeneratedCatalogMetadataDiffers(t *testing.T) {
tests := []struct {
name string
mutate func(*state.CatalogState)
}{
{
name: "source path",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].SourcePath = "summary.md"
},
},
{
name: "transform",
mutate: func(catalog *state.CatalogState) {
catalog.Outputs[0].Transform = "markdown_to_html_index"
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, destinationBackend, req := catalogGeneratedPlanRequest(t, config.WorkflowAdditive)
existing := matchingCatalogForRequest(t, req)
tt.mutate(&existing)
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action == ActionSkipSame {
t.Fatalf("plan action = %s, want write action after generated metadata change", plan.Action)
}
})
}
}
func TestBuildTransfersManagedPathOwnership(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
existing := baseCatalog(req)
existing.Outputs = []state.CatalogOutputFile{
catalogOutput(req, "reports", "web", "report.md", state.OutputKindSource, planCreatedAt),
}
writeCatalogState(t, destinationBackend, "", existing)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
written, ok := state.FindCatalogOutputByPath(plan.CatalogOutputsToWrite, "report.md")
if !ok {
t.Fatalf("written outputs = %#v, want report.md", plan.CatalogOutputsToWrite)
}
if written.PipelineID != "reports" || written.DestinationID != "archive" {
t.Fatalf("written owner = %s/%s, want reports/archive", written.PipelineID, written.DestinationID)
}
if !written.CreatedAt.Equal(planCreatedAt) || !written.UpdatedAt.Equal(planUpdatedAt) {
t.Fatalf("written times = %s/%s", written.CreatedAt, written.UpdatedAt)
}
if len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("retain=%#v delete=%#v, want no old record for overwritten path", plan.CatalogOutputsToRetain, plan.CatalogOutputsToDelete)
}
}
func TestBuildRejectsUnmanagedPlannedPathCollision(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
writeCatalogState(t, destinationBackend, "", baseCatalog(req))
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
plan, err := Build(context.Background(), req)
if err == nil {
t.Fatal("Build() error = nil, want unmanaged collision")
}
if plan.Action != ActionFailUnmanaged || !strings.Contains(err.Error(), "not managed by catalog state") {
t.Fatalf("plan action=%s error=%v, want unmanaged catalog collision", plan.Action, err)
}
}
func TestBuildForceReplacesUnmanagedPlannedPathCollision(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
req.Force = true
existing := baseCatalog(req)
writeCatalogState(t, destinationBackend, "", existing)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
}
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
}
}
func TestBuildRejectsNoStateNonEmptyDestinationWithoutForce(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "unmanaged")
plan, err := Build(context.Background(), req)
if err == nil {
t.Fatal("Build() error = nil, want unmanaged destination")
}
if plan.Action != ActionFailUnmanaged || !strings.Contains(err.Error(), "destination has content but no distributor state") {
t.Fatalf("plan action=%s error=%v, want unmanaged destination", plan.Action, err)
}
}
func TestBuildForceReplacesNoStateNonEmptyDestination(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
req.Force = true
testutil.WriteFakeFile(t, destinationBackend, "unplanned.txt", "unmanaged")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
}
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
}
}
func TestBuildPlansSupersededLegacyAdditive(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
legacyState := legacyStateDocument(2)
writeJSONState(t, destinationBackend, "", legacyState)
testutil.WriteFakeFile(t, destinationBackend, "report.md", "legacy")
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.SupersededLegacy == nil || plan.SupersededLegacy.SchemaVersion != 2 {
t.Fatalf("superseded legacy = %#v", plan.SupersededLegacy)
}
if plan.Action != ActionUpsertAdditive || plan.ClearDestinationRoot {
t.Fatalf("plan action=%s clear=%t, want additive overwrite without clear", plan.Action, plan.ClearDestinationRoot)
}
}
func TestBuildPlansSupersededLegacyReplacementClear(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowReplacement)
legacyState := legacyStateDocument(2)
writeJSONState(t, destinationBackend, "", legacyState)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceCatalog || !plan.ClearDestinationRoot {
t.Fatalf("plan action=%s clear=%t, want replacement clear", plan.Action, plan.ClearDestinationRoot)
}
}
func TestBuildRejectsInvalidOrFutureState(t *testing.T) {
tests := []struct {
name string
data string
}{
{name: "invalid json", data: `{"schema_version":`},
{name: "future schema", data: `{"schema_version":99}`},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
testutil.WriteFakeFile(t, destinationBackend, storage.StateFileName, tt.data)
plan, err := Build(context.Background(), req)
if err == nil {
t.Fatal("Build() error = nil, want conflict")
}
if plan.Action != ActionFailConflict {
t.Fatalf("plan action = %s, want %s", plan.Action, ActionFailConflict)
}
})
}
}
func TestBuildForceReplacesInvalidOrFutureState(t *testing.T) {
tests := []struct {
name string
data string
}{
{name: "invalid json", data: `{"schema_version":`},
{name: "future schema", data: `{"schema_version":99}`},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, config.WorkflowAdditive)
req.Force = true
testutil.WriteFakeFile(t, destinationBackend, storage.StateFileName, tt.data)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionForceReplace || !plan.Force || !plan.ClearDestinationRoot {
t.Fatalf("plan action=%s force=%t clear=%t, want forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot)
}
if len(plan.CatalogOutputsToWrite) != 2 || len(plan.CatalogOutputsToRetain) != 0 || len(plan.CatalogOutputsToDelete) != 0 {
t.Fatalf("catalog write=%d retain=%d delete=%d", len(plan.CatalogOutputsToWrite), len(plan.CatalogOutputsToRetain), len(plan.CatalogOutputsToDelete))
}
})
}
}
func TestBuildForceDoesNotChangeValidCatalogActions(t *testing.T) {
tests := []struct {
name string
workflow string
want Action
}{
{name: "additive", workflow: config.WorkflowAdditive, want: ActionUpsertAdditive},
{name: "replacement", workflow: config.WorkflowReplacement, want: ActionReplaceCatalog},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, destinationBackend, req := catalogPlanRequest(t, tt.workflow)
req.Force = true
writeCatalogState(t, destinationBackend, "", baseCatalog(req))
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != tt.want || plan.Force || plan.ClearDestinationRoot {
t.Fatalf("plan action=%s force=%t clear=%t, want %s without forced clear", plan.Action, plan.Force, plan.ClearDestinationRoot, tt.want)
}
})
}
}
func TestValidateRequestRejectsInvalidWorkflow(t *testing.T) {
sourceBackend, destinationBackend, req := catalogPlanRequest(t, "append")
req.SourceBackend = sourceBackend
req.DestinationBackend = destinationBackend
err := validateRequest(req)
if err == nil {
t.Fatal("validateRequest() error = nil, want invalid workflow")
}
if !strings.Contains(err.Error(), "destination.workflow") {
t.Fatalf("validateRequest() error = %v, want workflow context", err)
}
}
func catalogPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Backend, Request) {
t.Helper()
sourceBackend := fake.New()
destinationBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{})
return sourceBackend, destinationBackend, Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: "",
PathMapping: config.PathMappingPreserveRelative,
Publish: config.PublishPolicy{Source: true},
Workflow: workflow,
DistributorVersion: "test",
Now: planUpdatedAt,
}
}
func catalogGeneratedPlanRequest(t *testing.T, workflow string) (*fake.Backend, *fake.Backend, Request) {
t.Helper()
sourceBackend, destinationBackend, req := catalogPlanRequest(t, workflow)
data := []byte("<p>Generated</p>\n")
req.Publish = config.PublishPolicy{HTML: true}
req.Transform = config.Transform{MarkdownToHTML: &config.MarkdownToHTML{
Enabled: true,
Mode: config.TransformModeSidecar,
}}
req.Transformers = testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
Path: "report.html",
SourcePath: "report.md",
Transform: transform.MarkdownToHTML,
Data: data,
SHA256: bundle.FileDigest(data),
Size: int64(len(data)),
}}}}
return sourceBackend, destinationBackend, req
}
func matchingCatalogForRequest(t *testing.T, req Request) state.CatalogState {
t.Helper()
outputs, err := PlanOutputs(context.Background(), req)
if err != nil {
t.Fatalf("PlanOutputs() error = %v", err)
}
outputs, _, err = PlanLinks(req, outputs)
if err != nil {
t.Fatalf("PlanLinks() error = %v", err)
}
catalog := baseCatalog(req)
catalog.Outputs = catalogOutputsForPlan(req, outputs, nil, state.CurrentOwnerScope(req.PipelineID, req.DestinationID), planCreatedAt)
return catalog
}
func baseCatalog(req Request) state.CatalogState {
return state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: "previous",
CreatedAt: planCreatedAt,
UpdatedAt: planCreatedAt,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: []state.CatalogOutputFile{},
}
}
func catalogOutput(req Request, pipelineID, destinationID, path, kind string, createdAt time.Time) state.CatalogOutputFile {
sourceSHA := req.SourceBundle.Manifest.Files[0].SHA256
sourceSize := req.SourceBundle.Manifest.Files[0].Size
for _, file := range req.SourceBundle.Manifest.Files {
if file.Path == path {
sourceSHA = file.SHA256
sourceSize = file.Size
break
}
}
output := state.CatalogOutputFile{
Path: path,
PipelineID: pipelineID,
DestinationID: destinationID,
Source: state.CatalogSourceIdentity{
ID: req.SourceBundle.Manifest.ID,
Digest: req.SourceBundle.Manifest.Digest,
Created: req.SourceBundle.Manifest.Created,
},
Kind: kind,
SHA256: sourceSHA,
Size: sourceSize,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}
return output
}
func writeCatalogState(t *testing.T, backend *fake.Backend, relative string, catalog state.CatalogState) {
t.Helper()
writeJSONState(t, backend, relative, catalog)
}
func legacyStateDocument(schemaVersion int) map[string]int {
return map[string]int{"schema_version": schemaVersion}
}
func writeJSONState(t *testing.T, backend *fake.Backend, relative string, value any) {
t.Helper()
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
t.Fatalf("marshal state: %v", err)
}
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
}

View File

@@ -18,7 +18,11 @@ func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath
if parseErr != nil {
return state.DestinationStatus{StateErr: parseErr}, nil
}
return state.DestinationStatus{State: document.SingleOwner, SharedRoot: document.SharedRoot, HasContents: true}, nil
return state.DestinationStatus{
Catalog: document.Catalog,
SupersededLegacy: document.SupersededLegacy,
HasContents: true,
}, nil
}
if !storage.IsNotFound(err) {
return state.DestinationStatus{}, err

View File

@@ -1,399 +0,0 @@
package publish
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
func TestBuildSharedRootTreatsAbsentOwnerAsPublishable(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionPublishNew {
t.Fatalf("plan action = %s, want publish_new", plan.Action)
}
if plan.OwnerScope != state.CurrentOwnerScope("reports", "archive") {
t.Fatalf("owner scope = %#v, want reports/archive", plan.OwnerScope)
}
if got, want := len(plan.OtherOwnerOutputs), 1; got != want {
t.Fatalf("other owner output count = %d, want %d", got, want)
}
if got, want := len(plan.OwnerOutputsToWrite), 1; got != want {
t.Fatalf("owner output write count = %d, want %d", got, want)
}
if len(plan.OwnerOutputsToDelete) != 0 || len(plan.RetainedOwnerOutputs) != 0 {
t.Fatalf("delete=%#v retained=%#v, want none", plan.OwnerOutputsToDelete, plan.RetainedOwnerOutputs)
}
}
func TestBuildSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionReplaceOlder {
t.Fatalf("plan action = %s, want replace_older", plan.Action)
}
if got, want := sharedRootOutputPathList(plan.OwnerOutputsToDelete), "old.md"; got != want {
t.Fatalf("owner outputs to delete = %q, want %q", got, want)
}
if got, want := sharedRootOutputPathList(plan.OtherOwnerOutputs), "other/report.md"; got != want {
t.Fatalf("other owner outputs = %q, want %q", got, want)
}
if len(plan.RetainedOwnerOutputs) != 0 {
t.Fatalf("retained owner outputs = %#v, want none", plan.RetainedOwnerOutputs)
}
}
func TestBuildSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if got, want := sharedRootOutputPathList(plan.RetainedOwnerOutputs), "old.md"; got != want {
t.Fatalf("retained owner outputs = %q, want %q", got, want)
}
if len(plan.OwnerOutputsToDelete) != 0 {
t.Fatalf("owner outputs to delete = %#v, want none", plan.OwnerOutputsToDelete)
}
}
func TestBuildSharedRootRejectsOtherOwnerPathConflict(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
sharedRoot.Outputs[0].Path = "report.md"
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
t.Fatalf("Build() error = %v, want fail_conflict", err)
}
if plan.Action != ActionFailConflict {
t.Fatalf("plan action = %s, want fail_conflict", plan.Action)
}
}
func TestBuildSharedRootRejectsUnmanagedPathCollision(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
testutil.WriteFakeFile(t, destinationBackend, "bundle/report.md", "unmanaged")
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
t.Fatalf("Build() error = %v, want fail_unmanaged", err)
}
if plan.Action != ActionFailUnmanaged {
t.Fatalf("plan action = %s, want fail_unmanaged", plan.Action)
}
}
func TestExecuteSharedRootPublishesOwnerAndPreservesOtherOwners(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := len(destinationState.Owners), 2; got != want {
t.Fatalf("owner count = %d, want %d", got, want)
}
if got, want := destinationState.State.Mode, state.StateModeSharedRoot; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("other", "archive")); !ok {
t.Fatal("other owner missing from shared-root state")
}
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports", "archive")); !ok {
t.Fatal("current owner missing from shared-root state")
}
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
}
func TestExecuteSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
existing := sharedRootStateWithOwners(t, sourceBundle.Manifest, true)
writeFakeSharedRootState(t, destinationBackend, "bundle", existing)
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.md")
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
if !destinationState.CreatedAt.Equal(existing.CreatedAt) {
t.Fatalf("created_at = %s, want %s", destinationState.CreatedAt, existing.CreatedAt)
}
output, ok := findSharedRootOutputForTest(destinationState.Outputs, "report.md")
if !ok {
t.Fatal("report.md missing from shared-root outputs")
}
if !output.CreatedAt.Equal(existing.Outputs[1].CreatedAt) || !output.UpdatedAt.After(existing.Outputs[1].UpdatedAt) {
t.Fatalf("report.md timestamps = created:%s updated:%s, want preserved created and newer updated", output.CreatedAt, output.UpdatedAt)
}
}
func TestExecuteSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
})
destinationBackend := fake.New()
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
testutil.AssertFakeFile(t, destinationBackend, "bundle/old.md", "old")
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md,old.md"; got != want {
t.Fatalf("managed paths = %q, want %q", got, want)
}
}
func TestExecuteSharedRootForceReplaceDeletesOnlyBundlePath(t *testing.T) {
sourceBackend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
})
destinationBackend := fake.New()
testutil.WriteFakeFile(t, destinationBackend, "bundle/unmanaged.txt", "unmanaged")
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
req.Force = true
plan, err := Build(context.Background(), req)
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if plan.Action != ActionForceReplace {
t.Fatalf("plan action = %s, want force_replace", plan.Action)
}
if err := Execute(context.Background(), req, plan); err != nil {
t.Fatalf("Execute() error = %v", err)
}
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unmanaged.txt")
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
if got, want := len(destinationState.Owners), 1; got != want {
t.Fatalf("owner count = %d, want %d", got, want)
}
}
func sharedRootRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
return Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBundle: sourceBundle,
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: sourceBundle.RootRelativePath,
Publish: config.PublishPolicy{Source: true},
State: config.StatePolicy{Mode: config.StateModeSharedRoot},
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
Transfer: defaultTransfer(),
DistributorVersion: "test",
}
}
func writeFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string, sharedRoot state.SharedRootState) {
t.Helper()
data, err := json.MarshalIndent(sharedRoot, "", " ")
if err != nil {
t.Fatalf("marshal shared-root state: %v", err)
}
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
for _, output := range sharedRoot.Outputs {
path, err := storage.Join(relative, output.Path)
if err != nil {
t.Fatalf("join output path: %v", err)
}
testutil.WriteFakeFile(t, backend, path, "old")
}
}
func readFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string) state.SharedRootState {
t.Helper()
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
data, err := backend.ReadFile(context.Background(), statePath)
if err != nil {
t.Fatalf("read shared-root state: %v", err)
}
destinationState, err := state.ParseSharedRoot(data)
if err != nil {
t.Fatalf("parse shared-root state: %v", err)
}
return destinationState
}
func findSharedRootOutputForTest(outputs []state.SharedRootOutputFile, path string) (state.SharedRootOutputFile, bool) {
for _, output := range outputs {
if output.Path == path {
return output, true
}
}
return state.SharedRootOutputFile{}, false
}
func sharedRootStateWithOwners(t *testing.T, current bundle.Manifest, includeCurrent bool) state.SharedRootState {
t.Helper()
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
otherManifest := testutil.ValidManifest(testutil.BundleOptions{
ID: "other.source",
Files: []testutil.SourceFile{{Path: "other/report.md", Data: "# Other\n"}},
})
sharedRoot := state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: "test",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{{
Scope: state.CurrentOwnerScope("other", "archive"),
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: otherManifest},
}},
Outputs: []state.SharedRootOutputFile{{
Path: "other/report.md",
Kind: state.OutputKindSource,
SourcePath: "other/report.md",
SHA256: otherManifest.Files[0].SHA256,
Size: otherManifest.Files[0].Size,
Owner: state.CurrentOwnerScope("other", "archive"),
SourceID: otherManifest.ID,
SourceDigest: otherManifest.Digest,
SourceCreated: otherManifest.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}},
}
if !includeCurrent {
return sharedRoot
}
older := current
older.Created = older.Created.Add(-time.Hour)
older.Files = append([]bundle.ManifestFile(nil), current.Files...)
older.Files = append(older.Files, bundle.ManifestFile{
Path: "old.md",
SHA256: bundle.FileDigest([]byte("old\n")),
Size: int64(len("old\n")),
})
older.Digest = bundle.BundleDigest(older.Files)
scope := state.CurrentOwnerScope("reports", "archive")
sharedRoot.Owners = append(sharedRoot.Owners, state.OwnerRecord{
Scope: scope,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: older},
})
for _, file := range older.Files {
sharedRoot.Outputs = append(sharedRoot.Outputs, state.SharedRootOutputFile{
Path: file.Path,
Kind: state.OutputKindSource,
SourcePath: file.Path,
SHA256: file.SHA256,
Size: file.Size,
Owner: scope,
SourceID: older.ID,
SourceDigest: older.Digest,
SourceCreated: older.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
})
}
return sharedRoot
}
func sharedRootOutputPathList(outputs []state.SharedRootOutputFile) string {
paths := make([]string, 0, len(outputs))
for _, output := range outputs {
paths = append(paths, output.Path)
}
return strings.Join(paths, ",")
}

422
internal/state/catalog.go Normal file
View File

@@ -0,0 +1,422 @@
package state
import (
"bytes"
"encoding/json"
"fmt"
"io"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/link"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
)
type CatalogState struct {
SchemaVersion int
DistributorVersion string
CreatedAt time.Time
UpdatedAt time.Time
State StatePolicy
Outputs []CatalogOutputFile
}
type CatalogSourceIdentity struct {
ID string
Digest string
Created time.Time
}
type CatalogOutputFile struct {
Path string
PipelineID string
DestinationID string
Source CatalogSourceIdentity
Kind string
SourcePath string
Transform string
URL string
SHA256 string
Size int64
CreatedAt time.Time
UpdatedAt time.Time
}
type rawCatalogState struct {
SchemaVersion *int `json:"schema_version"`
DistributorVersion string `json:"distributor_version"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
State *rawStatePolicy `json:"state"`
Outputs []rawCatalogOutput `json:"outputs"`
}
type rawCatalogOutput struct {
Path *string `json:"path"`
PipelineID *string `json:"pipeline_id"`
DestinationID *string `json:"destination_id"`
Source *rawCatalogSourceIdentity `json:"source"`
Kind *string `json:"kind"`
SourcePath *string `json:"source_path"`
Transform *string `json:"transform"`
URL *string `json:"url"`
SHA256 *string `json:"sha256"`
Size *int64 `json:"size"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
}
type rawCatalogSourceIdentity struct {
ID *string `json:"id"`
Digest *string `json:"digest"`
Created *string `json:"created"`
}
func ParseCatalog(data []byte) (CatalogState, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var raw rawCatalogState
if err := decoder.Decode(&raw); err != nil {
return CatalogState{}, fmt.Errorf("parse distributor state: %w", err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return CatalogState{}, fmt.Errorf("parse distributor state: trailing data")
}
state, err := parseCatalogRaw(raw)
if err != nil {
return CatalogState{}, err
}
if err := ValidateCatalog(state); err != nil {
return CatalogState{}, err
}
return state, nil
}
func parseCatalogRaw(raw rawCatalogState) (CatalogState, error) {
if raw.SchemaVersion == nil {
return CatalogState{}, fmt.Errorf("state schema_version is required")
}
state := CatalogState{
SchemaVersion: *raw.SchemaVersion,
DistributorVersion: raw.DistributorVersion,
}
if state.SchemaVersion != CatalogSchemaVersion {
return CatalogState{}, fmt.Errorf("state schema_version must be %d", CatalogSchemaVersion)
}
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
if err != nil {
return CatalogState{}, err
}
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
if err != nil {
return CatalogState{}, err
}
state.CreatedAt = createdAt
state.UpdatedAt = updatedAt
if raw.State == nil || raw.State.Mode == "" {
return CatalogState{}, fmt.Errorf("state state.mode is required")
}
state.State.Mode = raw.State.Mode
if raw.Outputs == nil {
return CatalogState{}, fmt.Errorf("state outputs is required")
}
outputs, err := parseCatalogOutputs(raw.Outputs)
if err != nil {
return CatalogState{}, err
}
state.Outputs = outputs
return state, nil
}
func parseCatalogOutputs(rawOutputs []rawCatalogOutput) ([]CatalogOutputFile, error) {
outputs := make([]CatalogOutputFile, 0, len(rawOutputs))
for index, raw := range rawOutputs {
output, err := parseCatalogOutput(index, raw)
if err != nil {
return nil, err
}
outputs = append(outputs, output)
}
return outputs, nil
}
func parseCatalogOutput(index int, raw rawCatalogOutput) (CatalogOutputFile, error) {
if raw.Path == nil || *raw.Path == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
}
if raw.PipelineID == nil || *raw.PipelineID == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
}
if raw.DestinationID == nil || *raw.DestinationID == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
}
source, err := parseCatalogSourceIdentity(index, raw.Source)
if err != nil {
return CatalogOutputFile{}, err
}
if raw.Kind == nil || *raw.Kind == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
}
switch *raw.Kind {
case OutputKindSource:
if raw.SourcePath != nil {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].source_path is only valid for generated output", index)
}
if raw.Transform != nil {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].transform is only valid for generated output", index)
}
case OutputKindGenerated:
if raw.SourcePath == nil || *raw.SourcePath == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required for generated output", index)
}
if raw.Transform == nil || *raw.Transform == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].transform is required for generated output", index)
}
}
if raw.SHA256 == nil || *raw.SHA256 == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
}
if raw.Size == nil {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
}
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
if err != nil {
return CatalogOutputFile{}, err
}
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
if err != nil {
return CatalogOutputFile{}, err
}
output := CatalogOutputFile{
Path: *raw.Path,
PipelineID: *raw.PipelineID,
DestinationID: *raw.DestinationID,
Source: source,
Kind: *raw.Kind,
SHA256: *raw.SHA256,
Size: *raw.Size,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}
if raw.SourcePath != nil {
output.SourcePath = *raw.SourcePath
}
if raw.Transform != nil {
output.Transform = *raw.Transform
}
if raw.URL != nil {
if *raw.URL == "" {
return CatalogOutputFile{}, fmt.Errorf("state outputs[%d].url must not be empty", index)
}
output.URL = *raw.URL
}
return output, nil
}
func parseCatalogSourceIdentity(index int, raw *rawCatalogSourceIdentity) (CatalogSourceIdentity, error) {
if raw == nil {
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source is required", index)
}
if raw.ID == nil || *raw.ID == "" {
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source.id is required", index)
}
if raw.Digest == nil || *raw.Digest == "" {
return CatalogSourceIdentity{}, fmt.Errorf("state outputs[%d].source.digest is required", index)
}
created, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source.created", index), raw.Created)
if err != nil {
return CatalogSourceIdentity{}, err
}
return CatalogSourceIdentity{
ID: *raw.ID,
Digest: *raw.Digest,
Created: created,
}, nil
}
func (s CatalogState) CreatedAtString() string {
return s.CreatedAt.UTC().Format(time.RFC3339)
}
func (s CatalogState) UpdatedAtString() string {
return s.UpdatedAt.UTC().Format(time.RFC3339)
}
func (s CatalogSourceIdentity) CreatedString() string {
return s.Created.UTC().Format(time.RFC3339)
}
func (o CatalogOutputFile) CreatedAtString() string {
return o.CreatedAt.UTC().Format(time.RFC3339)
}
func (o CatalogOutputFile) UpdatedAtString() string {
return o.UpdatedAt.UTC().Format(time.RFC3339)
}
func (s CatalogState) MarshalJSON() ([]byte, error) {
type stateJSON struct {
SchemaVersion int `json:"schema_version"`
DistributorVersion string `json:"distributor_version,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
State StatePolicy `json:"state"`
Outputs []CatalogOutputFile `json:"outputs"`
}
return json.Marshal(stateJSON{
SchemaVersion: s.SchemaVersion,
DistributorVersion: s.DistributorVersion,
CreatedAt: s.CreatedAtString(),
UpdatedAt: s.UpdatedAtString(),
State: s.State,
Outputs: s.Outputs,
})
}
func (s CatalogSourceIdentity) MarshalJSON() ([]byte, error) {
type sourceJSON struct {
ID string `json:"id"`
Digest string `json:"digest"`
Created string `json:"created"`
}
return json.Marshal(sourceJSON{
ID: s.ID,
Digest: s.Digest,
Created: s.CreatedString(),
})
}
func (o CatalogOutputFile) MarshalJSON() ([]byte, error) {
type outputJSON struct {
Path string `json:"path"`
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
Source CatalogSourceIdentity `json:"source"`
Kind string `json:"kind"`
SourcePath string `json:"source_path,omitempty"`
Transform string `json:"transform,omitempty"`
URL string `json:"url,omitempty"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
return json.Marshal(outputJSON{
Path: o.Path,
PipelineID: o.PipelineID,
DestinationID: o.DestinationID,
Source: o.Source,
Kind: o.Kind,
SourcePath: o.SourcePath,
Transform: o.Transform,
URL: o.URL,
SHA256: o.SHA256,
Size: o.Size,
CreatedAt: o.CreatedAtString(),
UpdatedAt: o.UpdatedAtString(),
})
}
func ValidateCatalog(s CatalogState) error {
if s.SchemaVersion != CatalogSchemaVersion {
return fmt.Errorf("state schema_version must be %d", CatalogSchemaVersion)
}
if s.CreatedAt.IsZero() {
return fmt.Errorf("state created_at is required")
}
if s.UpdatedAt.IsZero() {
return fmt.Errorf("state updated_at is required")
}
if s.State.Mode != StateModeCatalog {
return fmt.Errorf("state state.mode must be %s", StateModeCatalog)
}
if s.Outputs == nil {
return fmt.Errorf("state outputs is required")
}
seenPaths := make(map[string]struct{}, len(s.Outputs))
for index, output := range s.Outputs {
if err := validateCatalogOutput(index, output); err != nil {
return err
}
if _, exists := seenPaths[output.Path]; exists {
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
}
seenPaths[output.Path] = struct{}{}
}
return nil
}
func validateCatalogOutput(index int, output CatalogOutputFile) error {
if err := storage.ValidatePath(output.Path); err != nil {
return fmt.Errorf("state outputs[%d].path: %w", index, err)
}
if output.PipelineID == "" {
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
}
if !config.IsSlugLikeID(output.PipelineID) {
return fmt.Errorf("state outputs[%d].pipeline_id must be a slug-like identifier", index)
}
if output.DestinationID == "" {
return fmt.Errorf("state outputs[%d].destination_id is required", index)
}
if !config.IsSlugLikeID(output.DestinationID) {
return fmt.Errorf("state outputs[%d].destination_id must be a slug-like identifier", index)
}
if err := validateCatalogSourceIdentity(index, output.Source); err != nil {
return err
}
switch output.Kind {
case OutputKindSource:
if output.SourcePath != "" {
return fmt.Errorf("state outputs[%d].source_path is only valid for generated output", index)
}
if output.Transform != "" {
return fmt.Errorf("state outputs[%d].transform is only valid for generated output", index)
}
case OutputKindGenerated:
if output.SourcePath == "" {
return fmt.Errorf("state outputs[%d].source_path is required for generated output", index)
}
if err := storage.ValidatePath(output.SourcePath); err != nil {
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
}
if output.Transform == "" {
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
}
default:
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
}
if output.URL != "" {
if err := link.ValidateHTTPURL(output.URL); err != nil {
return fmt.Errorf("state outputs[%d].url: %w", index, err)
}
}
if err := bundle.ValidateDigest(output.SHA256); err != nil {
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
}
if output.Size < 0 {
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
}
if output.CreatedAt.IsZero() {
return fmt.Errorf("state outputs[%d].created_at is required", index)
}
if output.UpdatedAt.IsZero() {
return fmt.Errorf("state outputs[%d].updated_at is required", index)
}
return nil
}
func validateCatalogSourceIdentity(index int, source CatalogSourceIdentity) error {
if source.ID == "" {
return fmt.Errorf("state outputs[%d].source.id is required", index)
}
if err := bundle.ValidateDigest(source.Digest); err != nil {
return fmt.Errorf("state outputs[%d].source.digest: %w", index, err)
}
if source.Created.IsZero() {
return fmt.Errorf("state outputs[%d].source.created is required", index)
}
return nil
}

View File

@@ -0,0 +1,398 @@
package state
import (
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
)
func TestParseCatalogState(t *testing.T) {
state, err := ParseCatalog([]byte(validCatalogStateJSON(t)))
if err != nil {
t.Fatalf("ParseCatalog() error = %v", err)
}
if got, want := state.SchemaVersion, CatalogSchemaVersion; got != want {
t.Fatalf("schema version = %d, want %d", got, want)
}
if got, want := state.CreatedAtString(), "2026-06-19T12:00:00Z"; got != want {
t.Fatalf("created_at = %q, want %q", got, want)
}
if got, want := state.UpdatedAtString(), "2026-06-19T12:05:00Z"; got != want {
t.Fatalf("updated_at = %q, want %q", got, want)
}
if got, want := state.State.Mode, StateModeCatalog; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if got, want := len(state.Outputs), 2; got != want {
t.Fatalf("output count = %d, want %d", got, want)
}
sourceOutput := state.Outputs[0]
if sourceOutput.SourcePath != "" || sourceOutput.Transform != "" {
t.Fatalf("source output source_path=%q transform=%q, want omitted", sourceOutput.SourcePath, sourceOutput.Transform)
}
generatedOutput := state.Outputs[1]
if generatedOutput.SourcePath != "report.md" || generatedOutput.Transform != "markdown_to_html" {
t.Fatalf("generated output source_path=%q transform=%q", generatedOutput.SourcePath, generatedOutput.Transform)
}
if got, want := generatedOutput.Source.CreatedString(), "2026-05-30T11:10:00Z"; got != want {
t.Fatalf("source created = %q, want %q", got, want)
}
}
func TestCatalogMarshalIsDeterministic(t *testing.T) {
data, err := json.Marshal(validCatalogState(t))
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
want := `{"schema_version":4,"distributor_version":"dev","created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z","state":{"mode":"catalog"},"outputs":[{"path":"report.md","pipeline_id":"reports","destination_id":"archive","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"source","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"},{"path":"report.html","pipeline_id":"reports","destination_id":"html","source":{"id":"weather.daily.brentwood.2026-05-30","digest":"sha256:c5590c36194c1307f20f85a64f2abe7b4769b2b1e4696d3753dba6f84011658f","created":"2026-05-30T11:10:00Z"},"kind":"generated","source_path":"report.md","transform":"markdown_to_html","url":"https://reports.example.com/report.html","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":128,"created_at":"2026-06-19T12:00:00Z","updated_at":"2026-06-19T12:05:00Z"}]}`
if string(data) != want {
t.Fatalf("json = %s, want %s", data, want)
}
}
func TestParseDocumentHandlesCatalogAndSupersededLegacy(t *testing.T) {
catalog, err := ParseDocument([]byte(validCatalogStateJSON(t)))
if err != nil {
t.Fatalf("ParseDocument(catalog) error = %v", err)
}
if catalog.Catalog == nil || catalog.SupersededLegacy != nil {
t.Fatalf("catalog document = %#v", catalog)
}
tests := map[string]string{
"schema 1": schemaOnlyStateJSON(legacySchemaVersion),
"schema 2": schemaOnlyStateJSON(legacyOwnerSchema),
"schema 3": schemaOnlyStateJSON(legacyMultiOwnerSchema),
}
for name, body := range tests {
t.Run(name, func(t *testing.T) {
document, err := ParseDocument([]byte(body))
if err != nil {
t.Fatalf("ParseDocument() error = %v", err)
}
if document.SupersededLegacy == nil || document.Catalog != nil {
t.Fatalf("document = %#v, want superseded legacy only", document)
}
if document.SupersededLegacy.SchemaVersion < legacySchemaVersion || document.SupersededLegacy.SchemaVersion >= CatalogSchemaVersion {
t.Fatalf("legacy schema version = %d, want 1 through 3", document.SupersededLegacy.SchemaVersion)
}
})
}
}
func TestParseDocumentRejectsUnsupportedFutureSchema(t *testing.T) {
body := strings.Replace(validCatalogStateJSON(t), `"schema_version": 4`, `"schema_version": 5`, 1)
_, err := ParseDocument([]byte(body))
assertStateErrorContains(t, err, "schema_version 5 is unsupported")
}
func TestParseDocumentRejectsTrailingData(t *testing.T) {
_, err := ParseDocument([]byte(validCatalogStateJSON(t) + "\n{}"))
assertStateErrorContains(t, err, "trailing data")
}
func TestParseCatalogRejectsMissingFields(t *testing.T) {
tests := map[string]func(map[string]any){
"schema_version": func(document map[string]any) {
delete(document, "schema_version")
},
"created_at": func(document map[string]any) {
delete(document, "created_at")
},
"updated_at": func(document map[string]any) {
delete(document, "updated_at")
},
"state": func(document map[string]any) {
delete(document, "state")
},
"outputs": func(document map[string]any) {
delete(document, "outputs")
},
"path": func(document map[string]any) {
delete(firstCatalogOutput(document), "path")
},
"pipeline_id": func(document map[string]any) {
delete(firstCatalogOutput(document), "pipeline_id")
},
"destination_id": func(document map[string]any) {
delete(firstCatalogOutput(document), "destination_id")
},
"source": func(document map[string]any) {
delete(firstCatalogOutput(document), "source")
},
"source id": func(document map[string]any) {
delete(firstCatalogSource(document), "id")
},
"source digest": func(document map[string]any) {
delete(firstCatalogSource(document), "digest")
},
"source created": func(document map[string]any) {
delete(firstCatalogSource(document), "created")
},
"kind": func(document map[string]any) {
delete(firstCatalogOutput(document), "kind")
},
"sha256": func(document map[string]any) {
delete(firstCatalogOutput(document), "sha256")
},
"size": func(document map[string]any) {
delete(firstCatalogOutput(document), "size")
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
document := catalogStateObject(t)
mutate(document)
_, err := ParseCatalog(mustMarshalCatalogObject(t, document))
assertStateErrorContains(t, err, "required")
})
}
}
func TestParseCatalogRejectsMalformedTimestamps(t *testing.T) {
tests := map[string]func(string) string{
"created_at": func(body string) string {
return strings.Replace(body, `"created_at": "2026-06-19T12:00:00Z"`, `"created_at": "June 19"`, 1)
},
"updated_at": func(body string) string {
return strings.Replace(body, `"updated_at": "2026-06-19T12:05:00Z"`, `"updated_at": "June 19"`, 1)
},
"source created": func(body string) string {
return strings.Replace(body, `"created": "2026-05-30T11:10:00Z"`, `"created": "May 30"`, 1)
},
"output created_at": func(body string) string {
return strings.Replace(body, ` "created_at": "2026-06-19T12:00:00Z"`, ` "created_at": "June 19"`, 1)
},
"output updated_at": func(body string) string {
return strings.Replace(body, ` "updated_at": "2026-06-19T12:05:00Z"`, ` "updated_at": "June 19"`, 1)
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
_, err := ParseCatalog([]byte(mutate(validCatalogStateJSON(t))))
assertStateErrorContains(t, err, "RFC3339")
})
}
}
func TestValidateCatalogRejectsInvalidOutputRecords(t *testing.T) {
tests := map[string]func(*CatalogState){
"duplicate path": func(s *CatalogState) {
s.Outputs[1].Path = s.Outputs[0].Path
},
"invalid output path": func(s *CatalogState) {
s.Outputs[0].Path = "../report.md"
},
"invalid pipeline id": func(s *CatalogState) {
s.Outputs[0].PipelineID = ".reports"
},
"invalid destination id": func(s *CatalogState) {
s.Outputs[0].DestinationID = ".archive"
},
"missing source id": func(s *CatalogState) {
s.Outputs[0].Source.ID = ""
},
"invalid source digest": func(s *CatalogState) {
s.Outputs[0].Source.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
},
"missing source created": func(s *CatalogState) {
s.Outputs[0].Source.Created = time.Time{}
},
"invalid kind": func(s *CatalogState) {
s.Outputs[0].Kind = "document"
},
"generated missing source path": func(s *CatalogState) {
s.Outputs[1].SourcePath = ""
},
"generated invalid source path": func(s *CatalogState) {
s.Outputs[1].SourcePath = "../report.md"
},
"generated missing transform": func(s *CatalogState) {
s.Outputs[1].Transform = ""
},
"source output source path": func(s *CatalogState) {
s.Outputs[0].SourcePath = "report.md"
},
"source output transform": func(s *CatalogState) {
s.Outputs[0].Transform = "markdown_to_html"
},
"invalid output digest": func(s *CatalogState) {
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
},
"negative size": func(s *CatalogState) {
s.Outputs[0].Size = -1
},
"invalid url": func(s *CatalogState) {
s.Outputs[1].URL = "file:///tmp/report.html"
},
"missing created at": func(s *CatalogState) {
s.Outputs[0].CreatedAt = time.Time{}
},
"missing updated at": func(s *CatalogState) {
s.Outputs[0].UpdatedAt = time.Time{}
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
state := validCatalogState(t)
mutate(&state)
if err := ValidateCatalog(state); err == nil {
t.Fatal("ValidateCatalog() error = nil, want error")
}
})
}
}
func TestParseCatalogRejectsForbiddenFields(t *testing.T) {
tests := map[string]string{
"owners": `"owners": [],`,
"sources": `"sources": [],`,
"workflow": `"workflow": "additive",`,
"source": `"source": {"manifest": {}},`,
"manifest": `"manifest": {},`,
"pipeline_id": `"pipeline_id": "reports",`,
"destination_id": `"destination_id": "archive",`,
"published_at": `"published_at": "2026-06-19T12:00:00Z",`,
}
for name, field := range tests {
t.Run(name, func(t *testing.T) {
body := strings.Replace(validCatalogStateJSON(t), `"created_at":`, field+"\n "+`"created_at":`, 1)
_, err := ParseCatalog([]byte(body))
assertStateErrorContains(t, err, "unknown field")
})
}
}
func TestParseCatalogRejectsForbiddenOutputFieldsForSourceOutput(t *testing.T) {
tests := map[string]string{
"source_path": `"source_path": "report.md",`,
"transform": `"transform": "markdown_to_html",`,
"empty url": `"url": "",`,
}
for name, field := range tests {
t.Run(name, func(t *testing.T) {
body := strings.Replace(validCatalogStateJSON(t), `"kind": "source",`, `"kind": "source",`+"\n "+field, 1)
_, err := ParseCatalog([]byte(body))
if err == nil {
t.Fatal("ParseCatalog() error = nil, want error")
}
})
}
}
func validCatalogStateJSON(t *testing.T) string {
t.Helper()
data, err := json.MarshalIndent(validCatalogState(t), "", " ")
if err != nil {
t.Fatalf("marshal catalog state: %v", err)
}
return string(data)
}
func catalogStateObject(t *testing.T) map[string]any {
t.Helper()
var document map[string]any
if err := json.Unmarshal([]byte(validCatalogStateJSON(t)), &document); err != nil {
t.Fatalf("unmarshal catalog state: %v", err)
}
return document
}
func firstCatalogOutput(document map[string]any) map[string]any {
outputs := document["outputs"].([]any)
return outputs[0].(map[string]any)
}
func firstCatalogSource(document map[string]any) map[string]any {
return firstCatalogOutput(document)["source"].(map[string]any)
}
func mustMarshalCatalogObject(t *testing.T, document map[string]any) []byte {
t.Helper()
data, err := json.Marshal(document)
if err != nil {
t.Fatalf("marshal catalog object: %v", err)
}
return data
}
func schemaOnlyStateJSON(schemaVersion int) string {
data, err := json.Marshal(map[string]int{"schema_version": schemaVersion})
if err != nil {
panic(err)
}
return string(data)
}
func assertStateErrorContains(t *testing.T, err error, substring string) {
t.Helper()
if err == nil {
t.Fatalf("error = nil, want substring %q", substring)
}
if !strings.Contains(err.Error(), substring) {
t.Fatalf("error = %v, want substring %q", err, substring)
}
}
func validManifest(t *testing.T) bundle.Manifest {
t.Helper()
files := []bundle.ManifestFile{{
Path: "report.md",
SHA256: bundle.FileDigest([]byte("# Report\nSunny.\n")),
Size: int64(len("# Report\nSunny.\n")),
}}
manifest := bundle.Manifest{
SchemaVersion: bundle.SchemaVersion,
ID: "weather.daily.brentwood.2026-05-30",
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
Files: files,
}
manifest.Digest = bundle.BundleDigest(manifest.Files)
return manifest
}
func validCatalogState(t *testing.T) CatalogState {
t.Helper()
manifest := validManifest(t)
createdAt := time.Date(2026, 6, 19, 12, 0, 0, 0, time.UTC)
updatedAt := time.Date(2026, 6, 19, 12, 5, 0, 0, time.UTC)
source := CatalogSourceIdentity{
ID: manifest.ID,
Digest: manifest.Digest,
Created: manifest.Created,
}
return CatalogState{
SchemaVersion: CatalogSchemaVersion,
DistributorVersion: "dev",
CreatedAt: createdAt,
UpdatedAt: updatedAt,
State: StatePolicy{Mode: StateModeCatalog},
Outputs: []CatalogOutputFile{{
Path: "report.md",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: OutputKindSource,
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}, {
Path: "report.html",
PipelineID: "reports",
DestinationID: "html",
Source: source,
Kind: OutputKindGenerated,
SourcePath: "report.md",
Transform: "markdown_to_html",
URL: "https://reports.example.com/report.html",
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Size: 128,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}},
}
}

View File

@@ -1,126 +0,0 @@
package state
import (
"fmt"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
)
type Outcome string
const (
OutcomeDestinationAbsent Outcome = "destination_absent"
OutcomeDestinationUnmanaged Outcome = "destination_unmanaged"
OutcomeInvalidState Outcome = "invalid_destination_state"
OutcomeIdentityMismatch Outcome = "destination_identity_mismatch"
OutcomeSameSource Outcome = "same_source_manifest"
OutcomeDestinationOlder Outcome = "destination_older"
OutcomeDestinationNewer Outcome = "destination_newer"
OutcomeSameCreatedConflict Outcome = "same_created_digest_conflict"
OutcomeDifferentSourceConflict Outcome = "different_source_conflict"
)
type DestinationStatus struct {
State *DistributorState
SharedRoot *SharedRootState
StateErr error
HasContents bool
}
type Comparison struct {
Outcome Outcome
Reason string
}
func CompareSharedRootOwner(source bundle.Manifest, scope OwnerScope, status DestinationStatus) Comparison {
if status.StateErr != nil {
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error()}
}
if status.SharedRoot != nil {
if err := ValidateSharedRoot(*status.SharedRoot); err != nil {
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
}
owner, ok := status.SharedRoot.Owner(scope)
if !ok {
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: fmt.Sprintf("destination owner %s/%s is absent", scope.PipelineID, scope.DestinationID)}
}
return compareManifests(source, owner.Source.Manifest)
}
if status.State != nil {
destinationState := *status.State
if err := Validate(destinationState); err != nil {
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
}
if destinationState.PipelineID != scope.PipelineID {
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationState.PipelineID, scope.PipelineID)}
}
if destinationState.DestinationID != scope.DestinationID {
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("destination id %q does not match %q", destinationState.DestinationID, scope.DestinationID)}
}
return compareManifests(source, destinationState.Source.Manifest)
}
if status.HasContents {
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state"}
}
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
}
func compareManifests(source, destination bundle.Manifest) Comparison {
if manifestsEqual(source, destination) {
return Comparison{Outcome: OutcomeSameSource, Reason: "destination source manifest matches source"}
}
if destination.ID != source.ID {
return Comparison{Outcome: OutcomeDifferentSourceConflict, Reason: "destination source id differs from source"}
}
if destination.Created.Before(source.Created) {
return Comparison{Outcome: OutcomeDestinationOlder, Reason: "destination source is older than source"}
}
if destination.Created.After(source.Created) {
return Comparison{Outcome: OutcomeDestinationNewer, Reason: "destination source is newer than source"}
}
if destination.Digest != source.Digest {
return Comparison{Outcome: OutcomeSameCreatedConflict, Reason: "destination source has same id and created time but different digest"}
}
return Comparison{Outcome: OutcomeInvalidState, Reason: "destination source differs from source without a supported comparison outcome"}
}
func Compare(source bundle.Manifest, pipelineID, destinationID string, status DestinationStatus) Comparison {
if status.StateErr != nil {
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error()}
}
if status.State == nil {
if status.HasContents {
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state"}
}
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
}
destinationState := *status.State
if err := Validate(destinationState); err != nil {
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
}
if destinationState.PipelineID != pipelineID {
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationState.PipelineID, pipelineID)}
}
if destinationState.DestinationID != destinationID {
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("destination id %q does not match %q", destinationState.DestinationID, destinationID)}
}
return compareManifests(source, destinationState.Source.Manifest)
}
func manifestsEqual(a, b bundle.Manifest) bool {
if a.SchemaVersion != b.SchemaVersion ||
a.ID != b.ID ||
a.Digest != b.Digest ||
!a.Created.Equal(b.Created) ||
len(a.Files) != len(b.Files) {
return false
}
for index := range a.Files {
if a.Files[index] != b.Files[index] {
return false
}
}
return true
}

View File

@@ -1,129 +0,0 @@
package state
import (
"errors"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
func TestCompareOutcomes(t *testing.T) {
source := validManifest(t)
tests := []struct {
name string
status DestinationStatus
want Outcome
}{
{
name: "destination absent",
status: DestinationStatus{},
want: OutcomeDestinationAbsent,
},
{
name: "destination unmanaged",
status: DestinationStatus{HasContents: true},
want: OutcomeDestinationUnmanaged,
},
{
name: "invalid destination state",
status: DestinationStatus{StateErr: errors.New("invalid json")},
want: OutcomeInvalidState,
},
{
name: "pipeline mismatch",
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
want: OutcomeIdentityMismatch,
},
{
name: "destination mismatch",
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
want: OutcomeIdentityMismatch,
},
{
name: "same source manifest",
status: DestinationStatus{State: withState(t, source, nil)},
want: OutcomeSameSource,
},
{
name: "destination older",
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
s.Source.Manifest.Created = source.Created.Add(-time.Hour)
})},
want: OutcomeDestinationOlder,
},
{
name: "destination newer",
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
s.Source.Manifest.Created = source.Created.Add(time.Hour)
})},
want: OutcomeDestinationNewer,
},
{
name: "same created digest conflict",
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
})},
want: OutcomeSameCreatedConflict,
},
{
name: "different source id",
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
s.Source.Manifest.ID = "other.source"
})},
want: OutcomeDifferentSourceConflict,
},
{
name: "invalid state object",
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
s.Outputs[0].Kind = "other"
})},
want: OutcomeInvalidState,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := Compare(source, "reports", "archive", tt.status)
if got.Outcome != tt.want {
t.Fatalf("Compare() outcome = %s reason=%q, want %s", got.Outcome, got.Reason, tt.want)
}
if got.Reason == "" {
t.Fatal("Compare() reason is empty")
}
})
}
}
func withState(t *testing.T, source bundle.Manifest, mutate func(*DistributorState)) *DistributorState {
t.Helper()
stateManifest := source
stateManifest.Files = append([]bundle.ManifestFile(nil), source.Files...)
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
state := DistributorState{
SchemaVersion: SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: StatePolicy{Mode: StateModeSingleOwner},
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: SourceState{Manifest: stateManifest},
Outputs: []OutputFile{{
Path: "report.md",
Kind: OutputKindSource,
SourcePath: "report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
}},
}
if mutate != nil {
mutate(&state)
}
return &state
}

View File

@@ -1,371 +0,0 @@
package state
import (
"bytes"
"encoding/json"
"fmt"
"io"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
const (
SchemaVersion = 2
SharedRootSchemaVersion = 3
legacySchemaVersion = 1
StateModeSingleOwner = config.StateModeSingleOwner
StateModeSharedRoot = config.StateModeSharedRoot
)
type DistributorState struct {
SchemaVersion int
DistributorVersion string
PipelineID string
DestinationID string
PublishedAt time.Time
CreatedAt time.Time
UpdatedAt time.Time
State StatePolicy
Reconciliation ReconciliationPolicy
Source SourceState
Links *LinkState
Outputs []OutputFile
}
type StatePolicy struct {
Mode string
}
type ReconciliationPolicy struct {
Mode string
}
type SourceState struct {
Manifest bundle.Manifest
}
type LinkState struct {
PrimaryURL string
}
type OutputFile struct {
Path string
Kind string
SourcePath string
Transform string
URL string
SHA256 string
Size int64
CreatedAt time.Time
UpdatedAt time.Time
}
type rawDistributorState struct {
SchemaVersion *int `json:"schema_version"`
DistributorVersion string `json:"distributor_version"`
PipelineID *string `json:"pipeline_id"`
DestinationID *string `json:"destination_id"`
PublishedAt *string `json:"published_at"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
State *rawStatePolicy `json:"state"`
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
Source *rawSourceState `json:"source"`
Links *rawLinkState `json:"links"`
Outputs []rawOutputFile `json:"outputs"`
}
type rawStatePolicy struct {
Mode string `json:"mode"`
}
type rawReconciliationPolicy struct {
Mode string `json:"mode"`
}
type rawSourceState struct {
Manifest json.RawMessage `json:"manifest"`
}
type rawLinkState struct {
PrimaryURL string `json:"primary_url"`
}
type rawOutputFile struct {
Path *string `json:"path"`
Kind *string `json:"kind"`
SourcePath *string `json:"source_path"`
Transform string `json:"transform"`
URL string `json:"url"`
SHA256 *string `json:"sha256"`
Size *int64 `json:"size"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
}
func Parse(data []byte) (DistributorState, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
var raw rawDistributorState
if err := decoder.Decode(&raw); err != nil {
return DistributorState{}, fmt.Errorf("parse distributor state: %w", err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return DistributorState{}, fmt.Errorf("parse distributor state: trailing data")
}
state, err := parseRaw(raw)
if err != nil {
return DistributorState{}, err
}
if err := Validate(state); err != nil {
return DistributorState{}, err
}
return state, nil
}
func parseRaw(raw rawDistributorState) (DistributorState, error) {
var state DistributorState
if raw.SchemaVersion == nil {
return DistributorState{}, fmt.Errorf("state schema_version is required")
}
state.SchemaVersion = *raw.SchemaVersion
if state.SchemaVersion != SchemaVersion && state.SchemaVersion != legacySchemaVersion {
return DistributorState{}, fmt.Errorf("state schema_version must be %d or %d", legacySchemaVersion, SchemaVersion)
}
legacy := state.SchemaVersion == legacySchemaVersion
state.DistributorVersion = raw.DistributorVersion
if raw.PipelineID == nil || *raw.PipelineID == "" {
return DistributorState{}, fmt.Errorf("state pipeline_id is required")
}
state.PipelineID = *raw.PipelineID
if raw.DestinationID == nil || *raw.DestinationID == "" {
return DistributorState{}, fmt.Errorf("state destination_id is required")
}
state.DestinationID = *raw.DestinationID
if raw.PublishedAt == nil || *raw.PublishedAt == "" {
return DistributorState{}, fmt.Errorf("state published_at is required")
}
publishedAt, err := time.Parse(time.RFC3339, *raw.PublishedAt)
if err != nil {
return DistributorState{}, fmt.Errorf("state published_at must be RFC3339: %w", err)
}
state.PublishedAt = publishedAt.UTC()
if legacy {
state.SchemaVersion = SchemaVersion
state.CreatedAt = state.PublishedAt
state.UpdatedAt = state.PublishedAt
state.State.Mode = StateModeSingleOwner
state.Reconciliation.Mode = config.ReconciliationModeReplace
} else {
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
if err != nil {
return DistributorState{}, err
}
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
if err != nil {
return DistributorState{}, err
}
state.CreatedAt = createdAt
state.UpdatedAt = updatedAt
if raw.State == nil || raw.State.Mode == "" {
return DistributorState{}, fmt.Errorf("state state.mode is required")
}
state.State.Mode = raw.State.Mode
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
return DistributorState{}, fmt.Errorf("state reconciliation.mode is required")
}
state.Reconciliation.Mode = raw.Reconciliation.Mode
}
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
return DistributorState{}, fmt.Errorf("state source.manifest is required")
}
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
if err != nil {
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
}
state.Source.Manifest = manifest
if raw.Links != nil {
state.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
}
if raw.Outputs == nil {
return DistributorState{}, fmt.Errorf("state outputs is required")
}
outputs, err := parseOutputs(raw.Outputs, legacy, state.PublishedAt)
if err != nil {
return DistributorState{}, err
}
state.Outputs = outputs
return state, nil
}
func parseRequiredTime(context string, raw *string) (time.Time, error) {
if raw == nil || *raw == "" {
return time.Time{}, fmt.Errorf("%s is required", context)
}
parsed, err := time.Parse(time.RFC3339, *raw)
if err != nil {
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", context, err)
}
return parsed.UTC(), nil
}
func parseOutputs(rawOutputs []rawOutputFile, legacy bool, publishedAt time.Time) ([]OutputFile, error) {
outputs := make([]OutputFile, 0, len(rawOutputs))
seen := make(map[string]struct{}, len(rawOutputs))
for index, raw := range rawOutputs {
output, err := parseOutput(index, raw, legacy, publishedAt)
if err != nil {
return nil, err
}
if _, exists := seen[output.Path]; exists {
return nil, fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
}
seen[output.Path] = struct{}{}
outputs = append(outputs, output)
}
return outputs, nil
}
func parseOutput(index int, raw rawOutputFile, legacy bool, publishedAt time.Time) (OutputFile, error) {
if raw.Path == nil || *raw.Path == "" {
return OutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
}
if raw.Kind == nil || *raw.Kind == "" {
return OutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
}
if raw.SourcePath == nil || *raw.SourcePath == "" {
return OutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
}
if raw.SHA256 == nil || *raw.SHA256 == "" {
return OutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
}
if raw.Size == nil {
return OutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
}
createdAt := publishedAt
updatedAt := publishedAt
if !legacy {
var err error
createdAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
if err != nil {
return OutputFile{}, err
}
updatedAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
if err != nil {
return OutputFile{}, err
}
}
return OutputFile{
Path: *raw.Path,
Kind: *raw.Kind,
SourcePath: *raw.SourcePath,
Transform: raw.Transform,
URL: raw.URL,
SHA256: *raw.SHA256,
Size: *raw.Size,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}, nil
}
func (s DistributorState) PublishedAtString() string {
return s.PublishedAt.UTC().Format(time.RFC3339)
}
func (s DistributorState) CreatedAtString() string {
return s.CreatedAt.UTC().Format(time.RFC3339)
}
func (s DistributorState) UpdatedAtString() string {
return s.UpdatedAt.UTC().Format(time.RFC3339)
}
func (o OutputFile) CreatedAtString() string {
return o.CreatedAt.UTC().Format(time.RFC3339)
}
func (o OutputFile) UpdatedAtString() string {
return o.UpdatedAt.UTC().Format(time.RFC3339)
}
func (s DistributorState) MarshalJSON() ([]byte, error) {
type sourceJSON struct {
Manifest bundle.Manifest `json:"manifest"`
}
type stateJSON struct {
SchemaVersion int `json:"schema_version"`
DistributorVersion string `json:"distributor_version,omitempty"`
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
PublishedAt string `json:"published_at"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
State StatePolicy `json:"state"`
Reconciliation ReconciliationPolicy `json:"reconciliation"`
Source sourceJSON `json:"source"`
Links *LinkState `json:"links,omitempty"`
Outputs []OutputFile `json:"outputs"`
}
return json.Marshal(stateJSON{
SchemaVersion: s.SchemaVersion,
DistributorVersion: s.DistributorVersion,
PipelineID: s.PipelineID,
DestinationID: s.DestinationID,
PublishedAt: s.PublishedAtString(),
CreatedAt: s.CreatedAtString(),
UpdatedAt: s.UpdatedAtString(),
State: s.State,
Reconciliation: s.Reconciliation,
Source: sourceJSON{Manifest: s.Source.Manifest},
Links: s.Links,
Outputs: s.Outputs,
})
}
func (p StatePolicy) MarshalJSON() ([]byte, error) {
type policyJSON struct {
Mode string `json:"mode"`
}
return json.Marshal(policyJSON{Mode: p.Mode})
}
func (p ReconciliationPolicy) MarshalJSON() ([]byte, error) {
type policyJSON struct {
Mode string `json:"mode"`
}
return json.Marshal(policyJSON{Mode: p.Mode})
}
func (l LinkState) MarshalJSON() ([]byte, error) {
type linkJSON struct {
PrimaryURL string `json:"primary_url,omitempty"`
}
return json.Marshal(linkJSON{PrimaryURL: l.PrimaryURL})
}
func (o OutputFile) MarshalJSON() ([]byte, error) {
type outputJSON struct {
Path string `json:"path"`
Kind string `json:"kind"`
SourcePath string `json:"source_path"`
Transform string `json:"transform,omitempty"`
URL string `json:"url,omitempty"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
return json.Marshal(outputJSON{
Path: o.Path,
Kind: o.Kind,
SourcePath: o.SourcePath,
Transform: o.Transform,
URL: o.URL,
SHA256: o.SHA256,
Size: o.Size,
CreatedAt: o.CreatedAt.UTC().Format(time.RFC3339),
UpdatedAt: o.UpdatedAt.UTC().Format(time.RFC3339),
})
}

View File

@@ -1,473 +0,0 @@
package state
import (
"encoding/json"
"os"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
func TestParseValidState(t *testing.T) {
state, err := Parse([]byte(validStateJSON(t)))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if state.SchemaVersion != SchemaVersion {
t.Fatalf("schema version = %d, want %d", state.SchemaVersion, SchemaVersion)
}
if state.PipelineID != "reports" || state.DestinationID != "archive" {
t.Fatalf("identity = %q/%q", state.PipelineID, state.DestinationID)
}
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
}
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("CreatedAtString() = %q, want %q", got, want)
}
if got, want := state.UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("UpdatedAtString() = %q, want %q", got, want)
}
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
if got, want := len(state.Outputs), 1; got != want {
t.Fatalf("output count = %d, want %d", got, want)
}
if got, want := state.Outputs[0].CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("output CreatedAtString() = %q, want %q", got, want)
}
}
func TestParseValidStateWithLinks(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"outputs": [`, `"links": {"primary_url": "https://reports.example.com/archive/report.md"},`+"\n "+`"outputs": [`, 1)
body = strings.Replace(body, `"source_path": "report.md",`, `"source_path": "report.md",`+"\n "+`"url": "https://reports.example.com/archive/report.md",`, 1)
state, err := Parse([]byte(body))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if state.Links == nil || state.Links.PrimaryURL != "https://reports.example.com/archive/report.md" {
t.Fatalf("links = %#v, want primary URL", state.Links)
}
if state.Outputs[0].URL != "https://reports.example.com/archive/report.md" {
t.Fatalf("output URL = %q", state.Outputs[0].URL)
}
}
func TestRemoveMissingOutputs(t *testing.T) {
state, err := Parse([]byte(validStateJSON(t)))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
state.Outputs = append(state.Outputs, OutputFile{
Path: "summary.txt",
Kind: OutputKindSource,
SourcePath: "summary.txt",
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Size: 10,
CreatedAt: state.CreatedAt,
UpdatedAt: state.UpdatedAt,
})
next, changed := RemoveMissingOutputs(state, []string{"summary.txt"})
if !changed {
t.Fatal("RemoveMissingOutputs() changed = false, want true")
}
if got, want := ManagedOutputPaths(next), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("paths = %#v, want %#v", got, want)
}
unchanged, changed := RemoveMissingOutputs(next, []string{"missing.txt"})
if changed {
t.Fatal("RemoveMissingOutputs() changed = true, want false")
}
if got, want := ManagedOutputPaths(unchanged), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("unchanged paths = %#v, want %#v", got, want)
}
}
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
state, err := Parse([]byte(body))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
}
}
func TestParseRejectsMissingFields(t *testing.T) {
tests := map[string]string{
"schema_version": `"schema_version"`,
"pipeline_id": `"pipeline_id"`,
"destination_id": `"destination_id"`,
"published_at": `"published_at"`,
"created_at": `"created_at"`,
"updated_at": `"updated_at"`,
"state": `"state"`,
"reconciliation": `"reconciliation"`,
"source": `"source"`,
"outputs": `"outputs"`,
}
for name, field := range tests {
t.Run(name, func(t *testing.T) {
body := strings.Replace(validStateJSON(t), field, `"missing_`+name+`"`, 1)
_, err := Parse([]byte(body))
assertStateErrorContains(t, err, "required")
})
}
}
func TestParseRejectsInvalidSchemaVersion(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"schema_version": 2`, `"schema_version": 3`, 1)
_, err := Parse([]byte(body))
assertStateErrorContains(t, err, "schema_version must be 1 or 2")
}
func TestParseLegacyStateInfersSingleOwnerDefaults(t *testing.T) {
state, err := Parse([]byte(legacyStateJSON(t)))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got, want := state.SchemaVersion, SchemaVersion; got != want {
t.Fatalf("schema version = %d, want normalized %d", got, want)
}
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("created_at = %q, want %q", got, want)
}
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
t.Fatalf("reconciliation mode = %q, want %q", got, want)
}
if got, want := state.Outputs[0].UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("output updated_at = %q, want %q", got, want)
}
}
func TestParseRejectsInvalidEmbeddedManifest(t *testing.T) {
body := validStateWithManifestJSON(t, strings.Replace(manifestJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1))
_, err := Parse([]byte(body))
assertStateErrorContains(t, err, "source.manifest")
}
func TestValidateRejectsInvalidEmbeddedManifest(t *testing.T) {
tests := map[string]func(bundle.Manifest) bundle.Manifest{
"schema version": func(manifest bundle.Manifest) bundle.Manifest {
manifest.SchemaVersion = 2
return manifest
},
"empty id": func(manifest bundle.Manifest) bundle.Manifest {
manifest.ID = ""
return manifest
},
"bad digest": func(manifest bundle.Manifest) bundle.Manifest {
manifest.Digest = "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"
return manifest
},
"zero created": func(manifest bundle.Manifest) bundle.Manifest {
manifest.Created = time.Time{}
return manifest
},
"empty files": func(manifest bundle.Manifest) bundle.Manifest {
manifest.Files = nil
manifest.Digest = bundle.BundleDigest(manifest.Files)
return manifest
},
"unsafe path": func(manifest bundle.Manifest) bundle.Manifest {
manifest.Files[0].Path = "../report.md"
manifest.Digest = bundle.BundleDigest(manifest.Files)
return manifest
},
"duplicate path": func(manifest bundle.Manifest) bundle.Manifest {
manifest.Files[1].Path = manifest.Files[0].Path
manifest.Digest = bundle.BundleDigest(manifest.Files)
return manifest
},
"negative size": func(manifest bundle.Manifest) bundle.Manifest {
manifest.Files[0].Size = -1
manifest.Digest = bundle.BundleDigest(manifest.Files)
return manifest
},
"digest mismatch": func(manifest bundle.Manifest) bundle.Manifest {
manifest.Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"
return manifest
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
source := mutate(validManifest(t))
state := *withState(t, validManifest(t), func(*DistributorState) {})
state.Source.Manifest = source
err := Validate(state)
assertStateErrorContains(t, err, "source.manifest")
})
}
}
func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
source := validManifest(t)
tests := map[string]func(*DistributorState){
"unsafe path": func(s *DistributorState) {
s.Outputs[0].Path = "../report.md"
},
"invalid kind": func(s *DistributorState) {
s.Outputs[0].Kind = "other"
},
"invalid source": func(s *DistributorState) {
s.Outputs[0].SourcePath = "../report.md"
},
"generated missing": func(s *DistributorState) {
s.Outputs[0].Kind = OutputKindGenerated
},
"invalid digest": func(s *DistributorState) {
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
},
"negative size": func(s *DistributorState) {
s.Outputs[0].Size = -1
},
"missing output created_at": func(s *DistributorState) {
s.Outputs[0].CreatedAt = time.Time{}
},
"missing output updated_at": func(s *DistributorState) {
s.Outputs[0].UpdatedAt = time.Time{}
},
"invalid output url": func(s *DistributorState) {
s.Outputs[0].URL = "file:///tmp/report.md"
},
"invalid primary url": func(s *DistributorState) {
s.Links = &LinkState{PrimaryURL: "file:///tmp/report.md"}
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
state := *withState(t, source, mutate)
err := Validate(state)
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
})
}
}
func TestValidateReportsURLFieldContext(t *testing.T) {
source := validManifest(t)
state := *withState(t, source, func(s *DistributorState) {
s.Links = &LinkState{PrimaryURL: "https://reports.example.com/archive#top"}
})
err := Validate(state)
assertStateErrorContains(t, err, "state links.primary_url")
assertStateErrorContains(t, err, "must not include a fragment")
state = *withState(t, source, func(s *DistributorState) {
s.Outputs[0].URL = "https://reports.example.com/archive?preview=1"
})
err = Validate(state)
assertStateErrorContains(t, err, "state outputs[0].url")
assertStateErrorContains(t, err, "must not include a query string")
}
func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "May 30"`, 1)
_, err := Parse([]byte(body))
assertStateErrorContains(t, err, "published_at must be RFC3339")
}
func TestParseRejectsMalformedCreatedTimestamp(t *testing.T) {
body := strings.Replace(validStateJSON(t), `"created_at": "2026-05-30T11:12:00Z"`, `"created_at": "May 30"`, 1)
_, err := Parse([]byte(body))
assertStateErrorContains(t, err, "created_at must be RFC3339")
}
func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
source := validManifest(t)
publishedAt := time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
state := DistributorState{
SchemaVersion: SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: StatePolicy{Mode: StateModeSingleOwner},
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: SourceState{Manifest: source},
Outputs: []OutputFile{{
Path: "report.md",
Kind: OutputKindSource,
SourcePath: "report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
}},
}
data, err := json.Marshal(state)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
if !strings.Contains(string(data), `"published_at":"2026-05-30T11:12:00Z"`) {
t.Fatalf("json = %s, want UTC RFC3339 published_at", data)
}
}
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
source := validManifest(t)
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
state := DistributorState{
SchemaVersion: SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: StatePolicy{Mode: StateModeSingleOwner},
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: SourceState{Manifest: source},
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
Outputs: []OutputFile{{
Path: "report.md",
Kind: OutputKindSource,
SourcePath: "report.md",
URL: "https://reports.example.com/archive/report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
}},
}
data, err := json.Marshal(state)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
if !strings.Contains(string(data), `"links":{"primary_url":"https://reports.example.com/archive/report.md"}`) {
t.Fatalf("json = %s, want links primary URL", data)
}
if !strings.Contains(string(data), `"url":"https://reports.example.com/archive/report.md"`) {
t.Fatalf("json = %s, want output URL", data)
}
}
func TestOutputHelpers(t *testing.T) {
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
updatedAt := createdAt.Add(time.Hour)
retained := []OutputFile{{
Path: "old.md",
Kind: OutputKindSource,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}, {
Path: "report.md",
Kind: OutputKindSource,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}}
projected := ProjectOutputs([]OutputProjection{{
Path: "report.md",
Kind: OutputKindSource,
}, {
Path: "new.md",
Kind: OutputKindSource,
}}, retained, updatedAt)
if got, ok := FindOutputByPath(projected, "report.md"); !ok || !got.CreatedAt.Equal(createdAt) || !got.UpdatedAt.Equal(updatedAt) {
t.Fatalf("projected report.md = %#v, want preserved created_at and updated updated_at", got)
}
merged, err := MergeOutputFiles(retained, projected)
if err != nil {
t.Fatalf("MergeOutputFiles() error = %v", err)
}
if got, want := len(merged), 3; got != want {
t.Fatalf("merged count = %d, want %d", got, want)
}
if got, want := ManagedOutputPaths(DistributorState{Outputs: merged}), []string{"old.md", "report.md", "new.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("managed paths = %#v, want %#v", got, want)
}
}
func validStateJSON(t *testing.T) string {
t.Helper()
return validStateWithManifestJSON(t, manifestJSON(t))
}
func validStateWithManifestJSON(t *testing.T, manifest string) string {
t.Helper()
return `{
"schema_version": 2,
"distributor_version": "dev",
"pipeline_id": "reports",
"destination_id": "archive",
"published_at": "2026-05-30T11:12:00Z",
"created_at": "2026-05-30T11:12:00Z",
"updated_at": "2026-05-30T11:12:00Z",
"state": {"mode": "single_owner"},
"reconciliation": {"mode": "replace"},
"source": {
"manifest": ` + manifest + `
},
"outputs": [
{
"path": "report.md",
"kind": "source",
"source_path": "report.md",
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
"size": 16,
"created_at": "2026-05-30T11:12:00Z",
"updated_at": "2026-05-30T11:12:00Z"
}
]
}`
}
func legacyStateJSON(t *testing.T) string {
t.Helper()
return strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(validStateJSON(t),
`"schema_version": 2`, `"schema_version": 1`, 1),
` "created_at": "2026-05-30T11:12:00Z",
`, "", 1),
` "updated_at": "2026-05-30T11:12:00Z",
`, "", 1),
` "state": {"mode": "single_owner"},
`, "", 1),
` "reconciliation": {"mode": "replace"},
`, "", 1),
`,
"created_at": "2026-05-30T11:12:00Z",
"updated_at": "2026-05-30T11:12:00Z"`, "", 1)
}
func manifestJSON(t *testing.T) string {
t.Helper()
data, err := os.ReadFile("../bundle/testdata/valid_bundle/manifest.json")
if err != nil {
t.Fatalf("read manifest fixture: %v", err)
}
return string(data)
}
func validManifest(t *testing.T) bundle.Manifest {
t.Helper()
manifest, err := bundle.ParseManifest([]byte(manifestJSON(t)))
if err != nil {
t.Fatalf("ParseManifest() error = %v", err)
}
return manifest
}
func assertStateErrorContains(t *testing.T, err error, want string) {
t.Helper()
if err == nil {
t.Fatalf("error = nil, want substring %q", want)
}
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want substring %q", err.Error(), want)
}
}

107
internal/state/document.go Normal file
View File

@@ -0,0 +1,107 @@
package state
import (
"bytes"
"encoding/json"
"fmt"
"io"
"time"
)
const (
legacyOwnerSchema = 2
legacyMultiOwnerSchema = 3
CatalogSchemaVersion = 4
legacySchemaVersion = 1
StateModeCatalog = "catalog"
OutputKindSource = "source"
OutputKindGenerated = "generated"
)
type StateDocument struct {
Catalog *CatalogState
SupersededLegacy *SupersededLegacyState
}
type DestinationStatus struct {
Catalog *CatalogState
SupersededLegacy *SupersededLegacyState
StateErr error
HasContents bool
}
type SupersededLegacyState struct {
SchemaVersion int
}
type StatePolicy struct {
Mode string
}
type OwnerScope struct {
PipelineID string
DestinationID string
}
type rawStatePolicy struct {
Mode string `json:"mode"`
}
func ParseDocument(data []byte) (StateDocument, error) {
schemaVersion, err := parseSchemaVersion(data)
if err != nil {
return StateDocument{}, err
}
switch schemaVersion {
case legacySchemaVersion, legacyOwnerSchema, legacyMultiOwnerSchema:
return StateDocument{SupersededLegacy: &SupersededLegacyState{SchemaVersion: schemaVersion}}, nil
case CatalogSchemaVersion:
catalog, err := ParseCatalog(data)
if err != nil {
return StateDocument{}, err
}
return StateDocument{Catalog: &catalog}, nil
default:
return StateDocument{}, fmt.Errorf("state schema_version %d is unsupported", schemaVersion)
}
}
func parseSchemaVersion(data []byte) (int, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
var raw struct {
SchemaVersion *int `json:"schema_version"`
}
if err := decoder.Decode(&raw); err != nil {
return 0, fmt.Errorf("parse distributor state: %w", err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return 0, fmt.Errorf("parse distributor state: trailing data")
}
if raw.SchemaVersion == nil {
return 0, fmt.Errorf("state schema_version is required")
}
return *raw.SchemaVersion, nil
}
func (p StatePolicy) MarshalJSON() ([]byte, error) {
type policyJSON struct {
Mode string `json:"mode"`
}
return json.Marshal(policyJSON{Mode: p.Mode})
}
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
}
func parseRequiredTime(field string, raw *string) (time.Time, error) {
if raw == nil || *raw == "" {
return time.Time{}, fmt.Errorf("%s is required", field)
}
parsed, err := time.Parse(time.RFC3339, *raw)
if err != nil {
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", field, err)
}
return parsed.UTC(), nil
}

View File

@@ -1,138 +1,25 @@
package state
import (
"fmt"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
)
type OutputProjection struct {
Path string
Kind string
SourcePath string
Transform string
URL string
SHA256 string
Size int64
}
func FindOutputByPath(outputs []OutputFile, path string) (OutputFile, bool) {
func FindCatalogOutputByPath(outputs []CatalogOutputFile, path string) (CatalogOutputFile, bool) {
for _, output := range outputs {
if output.Path == path {
return output, true
}
}
return OutputFile{}, false
return CatalogOutputFile{}, false
}
func MergeOutputFiles(retained, planned []OutputFile) ([]OutputFile, error) {
outputs := make([]OutputFile, 0, len(retained)+len(planned))
indexByPath := make(map[string]int, len(retained)+len(planned))
for _, output := range retained {
if _, exists := indexByPath[output.Path]; exists {
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
}
indexByPath[output.Path] = len(outputs)
outputs = append(outputs, output)
}
seenPlanned := make(map[string]struct{}, len(planned))
for _, output := range planned {
if _, exists := seenPlanned[output.Path]; exists {
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
}
seenPlanned[output.Path] = struct{}{}
if index, exists := indexByPath[output.Path]; exists {
outputs[index] = output
continue
}
indexByPath[output.Path] = len(outputs)
outputs = append(outputs, output)
}
return outputs, nil
}
func ManagedOutputPaths(s DistributorState) []string {
paths := make([]string, 0, len(s.Outputs))
for _, output := range s.Outputs {
paths = append(paths, output.Path)
}
return paths
}
func RemoveMissingOutputs(s DistributorState, missingPaths []string) (DistributorState, bool) {
if len(missingPaths) == 0 {
return s, false
}
missing := pathSet(missingPaths)
next := s
next.Outputs = make([]OutputFile, 0, len(s.Outputs))
changed := false
for _, output := range s.Outputs {
if _, remove := missing[output.Path]; remove {
changed = true
continue
}
next.Outputs = append(next.Outputs, output)
}
return next, changed
}
func ProjectOutputs(outputs []OutputProjection, existing []OutputFile, now time.Time) []OutputFile {
now = now.UTC()
files := make([]OutputFile, 0, len(outputs))
func CatalogOutputsForOwner(outputs []CatalogOutputFile, scope OwnerScope) []CatalogOutputFile {
selected := make([]CatalogOutputFile, 0, len(outputs))
for _, output := range outputs {
createdAt := now
if existingOutput, ok := FindOutputByPath(existing, output.Path); ok {
createdAt = existingOutput.CreatedAt
if output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
selected = append(selected, output)
}
files = append(files, OutputFile{
Path: output.Path,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
CreatedAt: createdAt,
UpdatedAt: now,
})
}
return files
return selected
}
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
}
func (s SharedRootState) Owner(scope OwnerScope) (OwnerRecord, bool) {
for _, owner := range s.Owners {
if owner.Scope == scope {
return owner, true
}
}
return OwnerRecord{}, false
}
func (s SharedRootState) SourceManifest(scope OwnerScope) (bundle.Manifest, bool) {
owner, ok := s.Owner(scope)
if !ok {
return bundle.Manifest{}, false
}
return owner.Source.Manifest, true
}
func (s SharedRootState) ManagedOutputPaths(scope OwnerScope) []string {
paths := make([]string, 0, len(s.Outputs))
for _, output := range s.Outputs {
if output.Owner == scope {
paths = append(paths, output.Path)
}
}
return paths
}
func (s SharedRootState) AllManagedOutputPaths() []string {
func CatalogManagedOutputPaths(s CatalogState) []string {
paths := make([]string, 0, len(s.Outputs))
for _, output := range s.Outputs {
paths = append(paths, output.Path)
@@ -140,16 +27,16 @@ func (s SharedRootState) AllManagedOutputPaths() []string {
return paths
}
func RemoveMissingSharedRootOwnerOutputs(s SharedRootState, scope OwnerScope, missingPaths []string) (SharedRootState, bool) {
func RemoveMissingCatalogOwnerOutputs(s CatalogState, scope OwnerScope, missingPaths []string) (CatalogState, bool) {
if len(missingPaths) == 0 {
return s, false
}
missing := pathSet(missingPaths)
next := s
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
next.Outputs = make([]CatalogOutputFile, 0, len(s.Outputs))
changed := false
for _, output := range s.Outputs {
if output.Owner == scope {
if output.PipelineID == scope.PipelineID && output.DestinationID == scope.DestinationID {
if _, remove := missing[output.Path]; remove {
changed = true
continue
@@ -160,13 +47,13 @@ func RemoveMissingSharedRootOwnerOutputs(s SharedRootState, scope OwnerScope, mi
return next, changed
}
func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (SharedRootState, bool) {
func RemoveMissingCatalogOutputs(s CatalogState, missingPaths []string) (CatalogState, bool) {
if len(missingPaths) == 0 {
return s, false
}
missing := pathSet(missingPaths)
next := s
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
next.Outputs = make([]CatalogOutputFile, 0, len(s.Outputs))
changed := false
for _, output := range s.Outputs {
if _, remove := missing[output.Path]; remove {
@@ -178,149 +65,6 @@ func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (S
return next, changed
}
func (s SharedRootState) OutputOwner(path string) (OwnerScope, bool) {
for _, output := range s.Outputs {
if output.Path == path {
return output.Owner, true
}
}
return OwnerScope{}, false
}
func (s SharedRootState) PathOwnershipConflict(scope OwnerScope, paths []string) (PathOwnershipConflict, bool) {
for _, path := range paths {
owner, exists := s.OutputOwner(path)
if exists && owner != scope {
return PathOwnershipConflict{Path: path, Owner: owner}, true
}
}
return PathOwnershipConflict{}, false
}
func ProjectSharedRootOutputs(outputs []OutputProjection, existing []SharedRootOutputFile, scope OwnerScope, source bundle.Manifest, now time.Time) []SharedRootOutputFile {
now = now.UTC()
files := make([]SharedRootOutputFile, 0, len(outputs))
for _, output := range outputs {
createdAt := now
if existingOutput, ok := findSharedRootOutput(existing, output.Path); ok && existingOutput.Owner == scope {
createdAt = existingOutput.CreatedAt
}
files = append(files, SharedRootOutputFile{
Path: output.Path,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
URL: output.URL,
SHA256: output.SHA256,
Size: output.Size,
Owner: scope,
SourceID: source.ID,
SourceDigest: source.Digest,
SourceCreated: source.Created,
CreatedAt: createdAt,
UpdatedAt: now,
})
}
return files
}
func ReplaceOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
}
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
return SharedRootState{}, err
}
next := s
next.Owners = upsertOwner(s.Owners, owner)
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
for _, output := range s.Outputs {
if output.Owner != scope {
next.Outputs = append(next.Outputs, output)
}
}
next.Outputs = append(next.Outputs, planned...)
return next, nil
}
func MergeOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
}
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
return SharedRootState{}, err
}
next := s
next.Owners = upsertOwner(s.Owners, owner)
outputs := make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
indexByPath := make(map[string]int, len(s.Outputs)+len(planned))
for _, output := range s.Outputs {
indexByPath[output.Path] = len(outputs)
outputs = append(outputs, output)
}
for _, output := range planned {
if index, exists := indexByPath[output.Path]; exists {
outputs[index] = output
continue
}
indexByPath[output.Path] = len(outputs)
outputs = append(outputs, output)
}
next.Outputs = outputs
return next, nil
}
func findSharedRootOutput(outputs []SharedRootOutputFile, path string) (SharedRootOutputFile, bool) {
for _, output := range outputs {
if output.Path == path {
return output, true
}
}
return SharedRootOutputFile{}, false
}
func sharedRootOutputPaths(outputs []SharedRootOutputFile) []string {
paths := make([]string, 0, len(outputs))
for _, output := range outputs {
paths = append(paths, output.Path)
}
return paths
}
func rejectDuplicateSharedRootOutputs(outputs []SharedRootOutputFile) error {
seen := make(map[string]struct{}, len(outputs))
for _, output := range outputs {
if _, exists := seen[output.Path]; exists {
return fmt.Errorf("state output path %q is duplicated", output.Path)
}
seen[output.Path] = struct{}{}
}
return nil
}
func validatePlannedSharedRootOutputs(scope OwnerScope, outputs []SharedRootOutputFile) error {
if err := rejectDuplicateSharedRootOutputs(outputs); err != nil {
return err
}
for _, output := range outputs {
if output.Owner != scope {
return fmt.Errorf("state output path %q is owned by %s/%s, not %s/%s", output.Path, output.Owner.PipelineID, output.Owner.DestinationID, scope.PipelineID, scope.DestinationID)
}
}
return nil
}
func upsertOwner(owners []OwnerRecord, owner OwnerRecord) []OwnerRecord {
next := append([]OwnerRecord(nil), owners...)
for index, existing := range next {
if existing.Scope == owner.Scope {
next[index] = owner
return next
}
}
return append(next, owner)
}
func pathSet(paths []string) map[string]struct{} {
set := make(map[string]struct{}, len(paths))
for _, path := range paths {

View File

@@ -22,24 +22,13 @@ type PrunePlan struct {
Preserved []PruneCandidate
}
func SingleOwnerPruneCandidates(s DistributorState) []PruneCandidate {
func CatalogPruneCandidates(s CatalogState, scope OwnerScope) []PruneCandidate {
candidates := make([]PruneCandidate, 0, len(s.Outputs))
for _, output := range s.Outputs {
candidates = append(candidates, PruneCandidate{
Path: output.Path,
UpdatedAt: output.UpdatedAt,
})
}
return candidates
}
func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandidate {
candidates := make([]PruneCandidate, 0, len(s.Outputs))
for _, output := range s.Outputs {
if output.Owner != scope {
if output.PipelineID != scope.PipelineID || output.DestinationID != scope.DestinationID {
continue
}
owner := output.Owner
owner := scope
candidates = append(candidates, PruneCandidate{
Path: output.Path,
UpdatedAt: output.UpdatedAt,

View File

@@ -74,19 +74,6 @@ func TestPlanPruneDeterministicTieBreaking(t *testing.T) {
}
}
func TestSharedRootPruneCandidatesPreserveOtherOwners(t *testing.T) {
sharedRoot := validSharedRootState(t)
scope := CurrentOwnerScope("reports", "archive")
candidates := SharedRootPruneCandidates(sharedRoot, scope)
if got, want := pruneCandidatePaths(candidates), "report.md"; got != want {
t.Fatalf("candidates = %q, want %q", got, want)
}
if candidates[0].Owner == nil || *candidates[0].Owner != scope {
t.Fatalf("candidate owner = %#v, want current owner", candidates[0].Owner)
}
}
func pruneCandidatePaths(candidates []PruneCandidate) string {
paths := make([]string, 0, len(candidates))
for _, candidate := range candidates {

View File

@@ -1,519 +0,0 @@
package state
import (
"bytes"
"encoding/json"
"fmt"
"io"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/link"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
)
type StateDocument struct {
SingleOwner *DistributorState
SharedRoot *SharedRootState
}
type SharedRootState struct {
SchemaVersion int
DistributorVersion string
CreatedAt time.Time
UpdatedAt time.Time
State StatePolicy
Owners []OwnerRecord
Outputs []SharedRootOutputFile
}
type OwnerScope struct {
PipelineID string
DestinationID string
}
type OwnerRecord struct {
Scope OwnerScope
Reconciliation ReconciliationPolicy
Source SourceState
Links *LinkState
}
type SharedRootOutputFile struct {
Path string
Kind string
SourcePath string
Transform string
URL string
SHA256 string
Size int64
Owner OwnerScope
SourceID string
SourceDigest string
SourceCreated time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
type PathOwnershipConflict struct {
Path string
Owner OwnerScope
}
type rawSharedRootState struct {
SchemaVersion *int `json:"schema_version"`
DistributorVersion string `json:"distributor_version"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
State *rawStatePolicy `json:"state"`
Owners []rawOwnerRecord `json:"owners"`
Outputs []rawSharedRootOutput `json:"outputs"`
}
type rawOwnerRecord struct {
PipelineID *string `json:"pipeline_id"`
DestinationID *string `json:"destination_id"`
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
Source *rawSourceState `json:"source"`
Links *rawLinkState `json:"links"`
}
type rawSharedRootOutput struct {
Path *string `json:"path"`
Kind *string `json:"kind"`
SourcePath *string `json:"source_path"`
Transform string `json:"transform"`
URL string `json:"url"`
SHA256 *string `json:"sha256"`
Size *int64 `json:"size"`
PipelineID *string `json:"pipeline_id"`
DestinationID *string `json:"destination_id"`
SourceID *string `json:"source_id"`
SourceDigest *string `json:"source_digest"`
SourceCreated *string `json:"source_created"`
CreatedAt *string `json:"created_at"`
UpdatedAt *string `json:"updated_at"`
}
func ParseDocument(data []byte) (StateDocument, error) {
schemaVersion, err := parseSchemaVersion(data)
if err != nil {
return StateDocument{}, err
}
if schemaVersion == SharedRootSchemaVersion {
sharedRoot, err := ParseSharedRoot(data)
if err != nil {
return StateDocument{}, err
}
return StateDocument{SharedRoot: &sharedRoot}, nil
}
singleOwner, err := Parse(data)
if err != nil {
return StateDocument{}, err
}
return StateDocument{SingleOwner: &singleOwner}, nil
}
func parseSchemaVersion(data []byte) (int, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
var raw struct {
SchemaVersion *int `json:"schema_version"`
}
if err := decoder.Decode(&raw); err != nil {
return 0, fmt.Errorf("parse distributor state: %w", err)
}
if raw.SchemaVersion == nil {
return 0, fmt.Errorf("state schema_version is required")
}
return *raw.SchemaVersion, nil
}
func ParseSharedRoot(data []byte) (SharedRootState, error) {
decoder := json.NewDecoder(bytes.NewReader(data))
var raw rawSharedRootState
if err := decoder.Decode(&raw); err != nil {
return SharedRootState{}, fmt.Errorf("parse distributor state: %w", err)
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
return SharedRootState{}, fmt.Errorf("parse distributor state: trailing data")
}
state, err := parseSharedRootRaw(raw)
if err != nil {
return SharedRootState{}, err
}
if err := ValidateSharedRoot(state); err != nil {
return SharedRootState{}, err
}
return state, nil
}
func parseSharedRootRaw(raw rawSharedRootState) (SharedRootState, error) {
if raw.SchemaVersion == nil {
return SharedRootState{}, fmt.Errorf("state schema_version is required")
}
state := SharedRootState{SchemaVersion: *raw.SchemaVersion}
if state.SchemaVersion != SharedRootSchemaVersion {
return SharedRootState{}, fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
}
state.DistributorVersion = raw.DistributorVersion
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
if err != nil {
return SharedRootState{}, err
}
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
if err != nil {
return SharedRootState{}, err
}
state.CreatedAt = createdAt
state.UpdatedAt = updatedAt
if raw.State == nil || raw.State.Mode == "" {
return SharedRootState{}, fmt.Errorf("state state.mode is required")
}
state.State.Mode = raw.State.Mode
if raw.Owners == nil {
return SharedRootState{}, fmt.Errorf("state owners is required")
}
owners, err := parseOwnerRecords(raw.Owners)
if err != nil {
return SharedRootState{}, err
}
state.Owners = owners
if raw.Outputs == nil {
return SharedRootState{}, fmt.Errorf("state outputs is required")
}
outputs, err := parseSharedRootOutputs(raw.Outputs)
if err != nil {
return SharedRootState{}, err
}
state.Outputs = outputs
return state, nil
}
func parseOwnerRecords(rawOwners []rawOwnerRecord) ([]OwnerRecord, error) {
owners := make([]OwnerRecord, 0, len(rawOwners))
for index, raw := range rawOwners {
owner, err := parseOwnerRecord(index, raw)
if err != nil {
return nil, err
}
owners = append(owners, owner)
}
return owners, nil
}
func parseOwnerRecord(index int, raw rawOwnerRecord) (OwnerRecord, error) {
if raw.PipelineID == nil || *raw.PipelineID == "" {
return OwnerRecord{}, fmt.Errorf("state owners[%d].pipeline_id is required", index)
}
if raw.DestinationID == nil || *raw.DestinationID == "" {
return OwnerRecord{}, fmt.Errorf("state owners[%d].destination_id is required", index)
}
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
return OwnerRecord{}, fmt.Errorf("state owners[%d].reconciliation.mode is required", index)
}
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest is required", index)
}
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
if err != nil {
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
}
owner := OwnerRecord{
Scope: OwnerScope{
PipelineID: *raw.PipelineID,
DestinationID: *raw.DestinationID,
},
Reconciliation: ReconciliationPolicy{Mode: raw.Reconciliation.Mode},
Source: SourceState{Manifest: manifest},
}
if raw.Links != nil {
owner.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
}
return owner, nil
}
func parseSharedRootOutputs(rawOutputs []rawSharedRootOutput) ([]SharedRootOutputFile, error) {
outputs := make([]SharedRootOutputFile, 0, len(rawOutputs))
for index, raw := range rawOutputs {
output, err := parseSharedRootOutput(index, raw)
if err != nil {
return nil, err
}
outputs = append(outputs, output)
}
return outputs, nil
}
func parseSharedRootOutput(index int, raw rawSharedRootOutput) (SharedRootOutputFile, error) {
if raw.Path == nil || *raw.Path == "" {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
}
if raw.Kind == nil || *raw.Kind == "" {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
}
if raw.SourcePath == nil || *raw.SourcePath == "" {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
}
if raw.SHA256 == nil || *raw.SHA256 == "" {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
}
if raw.Size == nil {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
}
if raw.PipelineID == nil || *raw.PipelineID == "" {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
}
if raw.DestinationID == nil || *raw.DestinationID == "" {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
}
if raw.SourceID == nil || *raw.SourceID == "" {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_id is required", index)
}
if raw.SourceDigest == nil || *raw.SourceDigest == "" {
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_digest is required", index)
}
sourceCreated, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source_created", index), raw.SourceCreated)
if err != nil {
return SharedRootOutputFile{}, err
}
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
if err != nil {
return SharedRootOutputFile{}, err
}
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
if err != nil {
return SharedRootOutputFile{}, err
}
return SharedRootOutputFile{
Path: *raw.Path,
Kind: *raw.Kind,
SourcePath: *raw.SourcePath,
Transform: raw.Transform,
URL: raw.URL,
SHA256: *raw.SHA256,
Size: *raw.Size,
Owner: OwnerScope{
PipelineID: *raw.PipelineID,
DestinationID: *raw.DestinationID,
},
SourceID: *raw.SourceID,
SourceDigest: *raw.SourceDigest,
SourceCreated: sourceCreated,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
}, nil
}
func (s SharedRootState) CreatedAtString() string {
return s.CreatedAt.UTC().Format(time.RFC3339)
}
func (s SharedRootState) UpdatedAtString() string {
return s.UpdatedAt.UTC().Format(time.RFC3339)
}
func (o SharedRootOutputFile) SourceCreatedString() string {
return o.SourceCreated.UTC().Format(time.RFC3339)
}
func (o SharedRootOutputFile) CreatedAtString() string {
return o.CreatedAt.UTC().Format(time.RFC3339)
}
func (o SharedRootOutputFile) UpdatedAtString() string {
return o.UpdatedAt.UTC().Format(time.RFC3339)
}
func (s SharedRootState) MarshalJSON() ([]byte, error) {
type stateJSON struct {
SchemaVersion int `json:"schema_version"`
DistributorVersion string `json:"distributor_version,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
State StatePolicy `json:"state"`
Owners []OwnerRecord `json:"owners"`
Outputs []SharedRootOutputFile `json:"outputs"`
}
return json.Marshal(stateJSON{
SchemaVersion: s.SchemaVersion,
DistributorVersion: s.DistributorVersion,
CreatedAt: s.CreatedAtString(),
UpdatedAt: s.UpdatedAtString(),
State: s.State,
Owners: s.Owners,
Outputs: s.Outputs,
})
}
func (o OwnerRecord) MarshalJSON() ([]byte, error) {
type sourceJSON struct {
Manifest bundle.Manifest `json:"manifest"`
}
type ownerJSON struct {
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
Reconciliation ReconciliationPolicy `json:"reconciliation"`
Source sourceJSON `json:"source"`
Links *LinkState `json:"links,omitempty"`
}
return json.Marshal(ownerJSON{
PipelineID: o.Scope.PipelineID,
DestinationID: o.Scope.DestinationID,
Reconciliation: o.Reconciliation,
Source: sourceJSON{Manifest: o.Source.Manifest},
Links: o.Links,
})
}
func (o SharedRootOutputFile) MarshalJSON() ([]byte, error) {
type outputJSON struct {
Path string `json:"path"`
Kind string `json:"kind"`
SourcePath string `json:"source_path"`
Transform string `json:"transform,omitempty"`
URL string `json:"url,omitempty"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
SourceID string `json:"source_id"`
SourceDigest string `json:"source_digest"`
SourceCreated string `json:"source_created"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
return json.Marshal(outputJSON{
Path: o.Path,
Kind: o.Kind,
SourcePath: o.SourcePath,
Transform: o.Transform,
URL: o.URL,
SHA256: o.SHA256,
Size: o.Size,
PipelineID: o.Owner.PipelineID,
DestinationID: o.Owner.DestinationID,
SourceID: o.SourceID,
SourceDigest: o.SourceDigest,
SourceCreated: o.SourceCreatedString(),
CreatedAt: o.CreatedAtString(),
UpdatedAt: o.UpdatedAtString(),
})
}
func ValidateSharedRoot(s SharedRootState) error {
if s.SchemaVersion != SharedRootSchemaVersion {
return fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
}
if s.CreatedAt.IsZero() {
return fmt.Errorf("state created_at is required")
}
if s.UpdatedAt.IsZero() {
return fmt.Errorf("state updated_at is required")
}
if s.State.Mode != StateModeSharedRoot {
return fmt.Errorf("state state.mode must be %s", StateModeSharedRoot)
}
if s.Owners == nil {
return fmt.Errorf("state owners is required")
}
owners := make(map[OwnerScope]OwnerRecord, len(s.Owners))
for index, owner := range s.Owners {
if err := validateOwnerRecord(index, owner); err != nil {
return err
}
if _, exists := owners[owner.Scope]; exists {
return fmt.Errorf("state owners[%d] duplicates owner %s/%s", index, owner.Scope.PipelineID, owner.Scope.DestinationID)
}
owners[owner.Scope] = owner
}
if s.Outputs == nil {
return fmt.Errorf("state outputs is required")
}
seenPaths := make(map[string]struct{}, len(s.Outputs))
for index, output := range s.Outputs {
if err := validateSharedRootOutput(index, output, owners); err != nil {
return err
}
if _, exists := seenPaths[output.Path]; exists {
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
}
seenPaths[output.Path] = struct{}{}
}
return nil
}
func validateOwnerRecord(index int, owner OwnerRecord) error {
if owner.Scope.PipelineID == "" {
return fmt.Errorf("state owners[%d].pipeline_id is required", index)
}
if owner.Scope.DestinationID == "" {
return fmt.Errorf("state owners[%d].destination_id is required", index)
}
if owner.Reconciliation.Mode != config.ReconciliationModeReplace && owner.Reconciliation.Mode != config.ReconciliationModeMerge {
return fmt.Errorf("state owners[%d].reconciliation.mode must be %s or %s", index, config.ReconciliationModeReplace, config.ReconciliationModeMerge)
}
if err := validateEmbeddedManifest(owner.Source.Manifest); err != nil {
return fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
}
if owner.Links != nil && owner.Links.PrimaryURL != "" {
if err := link.ValidateHTTPURL(owner.Links.PrimaryURL); err != nil {
return fmt.Errorf("state owners[%d].links.primary_url: %w", index, err)
}
}
return nil
}
func validateSharedRootOutput(index int, output SharedRootOutputFile, owners map[OwnerScope]OwnerRecord) error {
if err := storage.ValidatePath(output.Path); err != nil {
return fmt.Errorf("state outputs[%d].path: %w", index, err)
}
switch output.Kind {
case OutputKindSource, OutputKindGenerated:
default:
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
}
if err := storage.ValidatePath(output.SourcePath); err != nil {
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
}
if output.Kind == OutputKindGenerated && output.Transform == "" {
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
}
if output.URL != "" {
if err := link.ValidateHTTPURL(output.URL); err != nil {
return fmt.Errorf("state outputs[%d].url: %w", index, err)
}
}
if err := bundle.ValidateDigest(output.SHA256); err != nil {
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
}
if output.Size < 0 {
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
}
if output.Owner.PipelineID == "" {
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
}
if output.Owner.DestinationID == "" {
return fmt.Errorf("state outputs[%d].destination_id is required", index)
}
if _, exists := owners[output.Owner]; !exists {
return fmt.Errorf("state outputs[%d] references unknown owner %s/%s", index, output.Owner.PipelineID, output.Owner.DestinationID)
}
if output.SourceID == "" {
return fmt.Errorf("state outputs[%d].source_id is required", index)
}
if err := bundle.ValidateDigest(output.SourceDigest); err != nil {
return fmt.Errorf("state outputs[%d].source_digest: %w", index, err)
}
if output.SourceCreated.IsZero() {
return fmt.Errorf("state outputs[%d].source_created is required", index)
}
if output.CreatedAt.IsZero() {
return fmt.Errorf("state outputs[%d].created_at is required", index)
}
if output.UpdatedAt.IsZero() {
return fmt.Errorf("state outputs[%d].updated_at is required", index)
}
return nil
}

View File

@@ -1,320 +0,0 @@
package state
import (
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
)
func TestParseDocumentHandlesSingleOwnerAndSharedRoot(t *testing.T) {
singleOwner, err := ParseDocument([]byte(validStateJSON(t)))
if err != nil {
t.Fatalf("ParseDocument(single owner) error = %v", err)
}
if singleOwner.SingleOwner == nil || singleOwner.SharedRoot != nil {
t.Fatalf("single owner document = %#v", singleOwner)
}
sharedRoot, err := ParseDocument([]byte(validSharedRootStateJSON(t)))
if err != nil {
t.Fatalf("ParseDocument(shared root) error = %v", err)
}
if sharedRoot.SharedRoot == nil || sharedRoot.SingleOwner != nil {
t.Fatalf("shared root document = %#v", sharedRoot)
}
}
func TestParseSharedRootState(t *testing.T) {
state, err := ParseSharedRoot([]byte(validSharedRootStateJSON(t)))
if err != nil {
t.Fatalf("ParseSharedRoot() error = %v", err)
}
if got, want := state.SchemaVersion, SharedRootSchemaVersion; got != want {
t.Fatalf("schema version = %d, want %d", got, want)
}
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
t.Fatalf("created_at = %q, want %q", got, want)
}
if got, want := state.State.Mode, StateModeSharedRoot; got != want {
t.Fatalf("state mode = %q, want %q", got, want)
}
if got, want := len(state.Owners), 2; got != want {
t.Fatalf("owner count = %d, want %d", got, want)
}
if got, want := len(state.Outputs), 2; got != want {
t.Fatalf("output count = %d, want %d", got, want)
}
scope := CurrentOwnerScope("reports", "archive")
manifest, ok := state.SourceManifest(scope)
if !ok {
t.Fatal("SourceManifest() ok = false, want true")
}
if got, want := manifest.ID, validManifest(t).ID; got != want {
t.Fatalf("source manifest id = %q, want %q", got, want)
}
}
func TestParseSharedRootRejectsInvalidMetadata(t *testing.T) {
tests := map[string]func(string) string{
"schema": func(body string) string {
return strings.Replace(body, `"schema_version": 3`, `"schema_version": 2`, 1)
},
"state mode": func(body string) string {
return strings.Replace(body, `"mode": "shared_root"`, `"mode": "single_owner"`, 1)
},
"duplicate owner": func(body string) string {
return strings.Replace(body, `"destination_id": "html"`, `"destination_id": "archive"`, 1)
},
"unknown output owner": func(body string) string {
return strings.Replace(body, `"destination_id": "html",`, `"destination_id": "missing",`, 1)
},
"duplicate output": func(body string) string {
return strings.Replace(body, `"path": "report.html"`, `"path": "report.md"`, 1)
},
"invalid source digest": func(body string) string {
return strings.Replace(body, `"source_digest": "sha256:`, `"source_digest": "SHA256:`, 1)
},
"invalid owner link": func(body string) string {
return strings.Replace(body, `"primary_url": "https://reports.example.com/archive/report.md"`, `"primary_url": "file:///tmp/report.md"`, 1)
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
_, err := ParseSharedRoot([]byte(mutate(validSharedRootStateJSON(t))))
if err == nil {
t.Fatal("ParseSharedRoot() error = nil, want error")
}
})
}
}
func TestSharedRootMarshalNormalizesTimestamps(t *testing.T) {
state := validSharedRootState(t)
state.CreatedAt = time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
state.UpdatedAt = state.CreatedAt
state.Outputs[0].SourceCreated = state.CreatedAt
state.Outputs[0].CreatedAt = state.CreatedAt
state.Outputs[0].UpdatedAt = state.CreatedAt
data, err := json.Marshal(state)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
for _, want := range []string{
`"created_at":"2026-05-30T11:12:00Z"`,
`"updated_at":"2026-05-30T11:12:00Z"`,
`"source_created":"2026-05-30T11:12:00Z"`,
} {
if !strings.Contains(string(data), want) {
t.Fatalf("json = %s, want %s", data, want)
}
}
}
func TestSharedRootOutputHelpers(t *testing.T) {
state := validSharedRootState(t)
archive := CurrentOwnerScope("reports", "archive")
html := CurrentOwnerScope("reports", "html")
if got, want := state.ManagedOutputPaths(archive), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("archive paths = %#v, want %#v", got, want)
}
if got, want := state.AllManagedOutputPaths(), []string{"report.md", "report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("all paths = %#v, want %#v", got, want)
}
conflict, ok := state.PathOwnershipConflict(archive, []string{"report.html"})
if !ok || conflict.Owner != html {
t.Fatalf("conflict = %#v ok=%t, want html owner conflict", conflict, ok)
}
}
func TestRemoveMissingSharedRootOwnerOutputs(t *testing.T) {
state := validSharedRootState(t)
archive := CurrentOwnerScope("reports", "archive")
next, changed := RemoveMissingSharedRootOwnerOutputs(state, archive, []string{"report.md", "report.html"})
if !changed {
t.Fatal("RemoveMissingSharedRootOwnerOutputs() changed = false, want true")
}
if got, want := next.AllManagedOutputPaths(), []string{"report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("paths = %#v, want %#v", got, want)
}
if _, ok := next.OutputOwner("report.html"); !ok {
t.Fatal("report.html owner missing, want unrelated owner preserved")
}
}
func TestRemoveMissingSharedRootOutputs(t *testing.T) {
state := validSharedRootState(t)
next, changed := RemoveMissingSharedRootOutputs(state, []string{"report.md", "report.html"})
if !changed {
t.Fatal("RemoveMissingSharedRootOutputs() changed = false, want true")
}
if got := next.AllManagedOutputPaths(); len(got) != 0 {
t.Fatalf("paths = %#v, want none", got)
}
}
func TestSharedRootProjectAndMergeOwnerOutputs(t *testing.T) {
state := validSharedRootState(t)
archive := CurrentOwnerScope("reports", "archive")
owner, ok := state.Owner(archive)
if !ok {
t.Fatal("Owner() ok = false, want true")
}
now := time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC)
planned := ProjectSharedRootOutputs([]OutputProjection{{
Path: "report.md",
Kind: OutputKindSource,
SourcePath: "report.md",
SHA256: validManifest(t).Files[0].SHA256,
Size: validManifest(t).Files[0].Size,
}, {
Path: "summary.txt",
Kind: OutputKindSource,
SourcePath: "summary.txt",
SHA256: validManifest(t).Files[1].SHA256,
Size: validManifest(t).Files[1].Size,
}}, state.Outputs, archive, validManifest(t), now)
merged, err := MergeOwnerOutputs(state, archive, owner, planned)
if err != nil {
t.Fatalf("MergeOwnerOutputs() error = %v", err)
}
if got, want := merged.AllManagedOutputPaths(), []string{"report.md", "report.html", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("merged paths = %#v, want %#v", got, want)
}
if merged.Outputs[0].CreatedAt.Equal(now) {
t.Fatalf("merged updated output created_at = %s, want preserved timestamp", merged.Outputs[0].CreatedAt)
}
if !merged.Outputs[0].UpdatedAt.Equal(now) {
t.Fatalf("merged updated output updated_at = %s, want %s", merged.Outputs[0].UpdatedAt, now)
}
replaced, err := ReplaceOwnerOutputs(state, archive, owner, planned)
if err != nil {
t.Fatalf("ReplaceOwnerOutputs() error = %v", err)
}
if got, want := replaced.AllManagedOutputPaths(), []string{"report.html", "report.md", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("replaced paths = %#v, want %#v", got, want)
}
}
func TestSharedRootOwnerOutputHelpersRejectConflicts(t *testing.T) {
state := validSharedRootState(t)
archive := CurrentOwnerScope("reports", "archive")
owner, ok := state.Owner(archive)
if !ok {
t.Fatal("Owner() ok = false, want true")
}
planned := []SharedRootOutputFile{{
Path: "report.html",
Kind: OutputKindSource,
Owner: archive,
CreatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
UpdatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
}}
if _, err := MergeOwnerOutputs(state, archive, owner, planned); err == nil {
t.Fatal("MergeOwnerOutputs() error = nil, want owner conflict")
}
if _, err := ReplaceOwnerOutputs(state, archive, owner, planned); err == nil {
t.Fatal("ReplaceOwnerOutputs() error = nil, want owner conflict")
}
}
func TestCompareSharedRootOwnerScopesCurrentOwner(t *testing.T) {
state := validSharedRootState(t)
source := validManifest(t)
source.Created = source.Created.Add(time.Hour)
scope := CurrentOwnerScope("reports", "archive")
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{SharedRoot: &state, HasContents: true})
if comparison.Outcome != OutcomeDestinationOlder {
t.Fatalf("comparison = %#v, want destination older for current owner", comparison)
}
missing := CompareSharedRootOwner(source, CurrentOwnerScope("missing", "archive"), DestinationStatus{SharedRoot: &state, HasContents: true})
if missing.Outcome != OutcomeDestinationAbsent {
t.Fatalf("missing owner comparison = %#v, want destination absent", missing)
}
}
func TestCompareSharedRootOwnerAcceptsMatchingSingleOwnerState(t *testing.T) {
singleOwner, err := Parse([]byte(validStateJSON(t)))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
source := singleOwner.Source.Manifest
source.Created = source.Created.Add(time.Hour)
scope := CurrentOwnerScope(singleOwner.PipelineID, singleOwner.DestinationID)
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{State: &singleOwner, HasContents: true})
if comparison.Outcome != OutcomeDestinationOlder {
t.Fatalf("comparison = %#v, want destination older", comparison)
}
}
func validSharedRootStateJSON(t *testing.T) string {
t.Helper()
data, err := json.MarshalIndent(validSharedRootState(t), "", " ")
if err != nil {
t.Fatalf("marshal shared root state: %v", err)
}
return string(data)
}
func validSharedRootState(t *testing.T) SharedRootState {
t.Helper()
source := validManifest(t)
htmlSource := source
htmlSource.Files = append([]bundle.ManifestFile(nil), source.Files...)
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
return SharedRootState{
SchemaVersion: SharedRootSchemaVersion,
DistributorVersion: "dev",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: StatePolicy{Mode: StateModeSharedRoot},
Owners: []OwnerRecord{{
Scope: CurrentOwnerScope("reports", "archive"),
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: SourceState{Manifest: source},
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
}, {
Scope: CurrentOwnerScope("reports", "html"),
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
Source: SourceState{Manifest: htmlSource},
}},
Outputs: []SharedRootOutputFile{{
Path: "report.md",
Kind: OutputKindSource,
SourcePath: "report.md",
SHA256: source.Files[0].SHA256,
Size: source.Files[0].Size,
Owner: CurrentOwnerScope("reports", "archive"),
SourceID: source.ID,
SourceDigest: source.Digest,
SourceCreated: source.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}, {
Path: "report.html",
Kind: OutputKindGenerated,
SourcePath: "report.md",
Transform: "markdown_to_html",
URL: "https://reports.example.com/html/report.html",
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Size: 128,
Owner: CurrentOwnerScope("reports", "html"),
SourceID: htmlSource.ID,
SourceDigest: htmlSource.Digest,
SourceCreated: htmlSource.Created,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}},
}
}

View File

@@ -1,103 +0,0 @@
package state
import (
"fmt"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/link"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
)
const (
OutputKindSource = "source"
OutputKindGenerated = "generated"
)
func Validate(s DistributorState) error {
if s.SchemaVersion != SchemaVersion {
return fmt.Errorf("state schema_version must be %d", SchemaVersion)
}
if s.PipelineID == "" {
return fmt.Errorf("state pipeline_id is required")
}
if s.DestinationID == "" {
return fmt.Errorf("state destination_id is required")
}
if s.PublishedAt.IsZero() {
return fmt.Errorf("state published_at is required")
}
if s.CreatedAt.IsZero() {
return fmt.Errorf("state created_at is required")
}
if s.UpdatedAt.IsZero() {
return fmt.Errorf("state updated_at is required")
}
if s.State.Mode != StateModeSingleOwner {
return fmt.Errorf("state state.mode must be %s", StateModeSingleOwner)
}
if s.Reconciliation.Mode != config.ReconciliationModeReplace && s.Reconciliation.Mode != config.ReconciliationModeMerge {
return fmt.Errorf("state reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
}
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
return fmt.Errorf("state source.manifest: %w", err)
}
if s.Links != nil && s.Links.PrimaryURL != "" {
if err := link.ValidateHTTPURL(s.Links.PrimaryURL); err != nil {
return fmt.Errorf("state links.primary_url: %w", err)
}
}
if s.Outputs == nil {
return fmt.Errorf("state outputs is required")
}
seen := make(map[string]struct{}, len(s.Outputs))
for index, output := range s.Outputs {
if err := validateOutput(index, output); err != nil {
return err
}
if _, exists := seen[output.Path]; exists {
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
}
seen[output.Path] = struct{}{}
}
return nil
}
func validateEmbeddedManifest(manifest bundle.Manifest) error {
return bundle.ValidateManifest(manifest)
}
func validateOutput(index int, output OutputFile) error {
if err := storage.ValidatePath(output.Path); err != nil {
return fmt.Errorf("state outputs[%d].path: %w", index, err)
}
switch output.Kind {
case OutputKindSource, OutputKindGenerated:
default:
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
}
if err := storage.ValidatePath(output.SourcePath); err != nil {
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
}
if output.Kind == OutputKindGenerated && output.Transform == "" {
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
}
if output.URL != "" {
if err := link.ValidateHTTPURL(output.URL); err != nil {
return fmt.Errorf("state outputs[%d].url: %w", index, err)
}
}
if err := bundle.ValidateDigest(output.SHA256); err != nil {
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
}
if output.Size < 0 {
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
}
if output.CreatedAt.IsZero() {
return fmt.Errorf("state outputs[%d].created_at is required", index)
}
if output.UpdatedAt.IsZero() {
return fmt.Errorf("state outputs[%d].updated_at is required", index)
}
return nil
}

View File

@@ -11,8 +11,6 @@ import (
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
)
@@ -33,13 +31,6 @@ type BundleOptions struct {
ExtraFiles []SourceFile
}
type DestinationStateOptions struct {
PipelineID string
DestinationID string
DistributorVersion string
PublishedAt time.Time
}
func DefaultSourceFiles() []SourceFile {
return []SourceFile{
{Path: "report.md", Data: "# Report\nSunny.\n"},
@@ -133,28 +124,6 @@ func AssertFakeMissing(t testing.TB, backend *fake.Backend, path string) {
}
}
func WriteFakeDestinationState(t testing.TB, backend *fake.Backend, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
t.Helper()
destinationState := DestinationState(manifest, opts)
data, err := json.MarshalIndent(destinationState, "", " ")
if err != nil {
t.Fatalf("marshal destination state: %v", err)
}
statePath, err := storage.StatePath(relative)
if err != nil {
t.Fatalf("state path: %v", err)
}
WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
for _, output := range destinationState.Outputs {
path, err := storage.Join(relative, output.Path)
if err != nil {
t.Fatalf("join output path: %v", err)
}
WriteFakeFile(t, backend, path, "old")
}
return destinationState
}
func WriteMinimalLocalConfig(t testing.TB, sourceRoot, destinationRoot string) string {
t.Helper()
return writeConfigFile(t, `
@@ -318,47 +287,6 @@ pipelines:
`)
}
func WriteDestinationState(t testing.TB, root, relative string, manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
t.Helper()
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
t.Fatalf("mkdir destination bundle: %v", err)
}
destinationState := DestinationState(manifest, opts)
writeJSONFile(t, filepath.Join(bundleRoot, storage.StateFileName), destinationState)
return destinationState
}
func DestinationState(manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
publishedAt := defaultPublishedAt(opts.PublishedAt)
return state.DistributorState{
SchemaVersion: state.SchemaVersion,
DistributorVersion: opts.DistributorVersion,
PipelineID: defaultString(opts.PipelineID, "reports"),
DestinationID: defaultString(opts.DestinationID, "archive"),
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
Outputs: sourceOutputs(manifest, publishedAt),
}
}
func ReadDestinationState(t testing.TB, path string) state.DistributorState {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read destination state: %v", err)
}
destinationState, err := state.Parse(data)
if err != nil {
t.Fatalf("parse destination state: %v", err)
}
return destinationState
}
func AssertFile(t testing.TB, path, want string) {
t.Helper()
data, err := os.ReadFile(path)
@@ -392,22 +320,6 @@ func sourceFiles(opts BundleOptions) []SourceFile {
return files
}
func sourceOutputs(manifest bundle.Manifest, publishedAt time.Time) []state.OutputFile {
outputs := make([]state.OutputFile, 0, len(manifest.Files))
for _, file := range manifest.Files {
outputs = append(outputs, state.OutputFile{
Path: file.Path,
Kind: state.OutputKindSource,
SourcePath: file.Path,
SHA256: file.SHA256,
Size: file.Size,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
})
}
return outputs
}
func writeConfigFile(t testing.TB, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
@@ -455,10 +367,3 @@ func defaultCreated(value time.Time) time.Time {
}
return value
}
func defaultPublishedAt(value time.Time) time.Time {
if value.IsZero() {
return time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
}
return value
}

View File

@@ -123,7 +123,7 @@ func (t *Transformer) render(ctx context.Context, req transform.Request, sourceF
if err := t.renderer.Convert(data, &rendered); err != nil {
return nil, fmt.Errorf("render markdown source %q: %w", sourceFile, err)
}
return wrapHTML(rendered.Bytes()), nil
return wrapHTML(rendered.Bytes(), req.Markdown.CssHref), nil
}
func markdownMode(mode string) string {

View File

@@ -40,6 +40,43 @@ func TestGenerateMarkdownSidecar(t *testing.T) {
}
}
func TestGenerateMarkdownWithoutCSSHrefPreservesWrapper(t *testing.T) {
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
outputs, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle})
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
want := "<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<title></title>\n</head>\n<body>\n<h1>Title</h1>\n<p>Hello.</p>\n</body>\n</html>\n"
if got := string(outputs[0].Data); got != want {
t.Fatalf("html = %q, want existing wrapper %q", got, want)
}
}
func TestGenerateMarkdownSidecarWithCSSHref(t *testing.T) {
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
outputs, err := New().Generate(context.Background(), transform.Request{
SourceBackend: backend,
SourceBundle: sourceBundle,
Markdown: transform.MarkdownOptions{CssHref: "/assets/report.css?v=1&theme=main"},
})
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
output := outputs[0]
html := string(output.Data)
wantLink := "<meta charset=\"utf-8\">\n<link rel=\"stylesheet\" href=\"/assets/report.css?v=1&amp;theme=main\">\n<title></title>"
if !strings.Contains(html, wantLink) {
t.Fatalf("html = %q, want stylesheet link %q", html, wantLink)
}
if output.SHA256 != bundle.FileDigest(output.Data) || output.Size != int64(len(output.Data)) {
t.Fatalf("digest/size metadata = %s/%d", output.SHA256, output.Size)
}
}
func TestGenerateMarkdownIndexExplicitInput(t *testing.T) {
backend := fake.New()
sourceBundle := testutil.WriteFakeSourceBundle(t, backend, "", testutil.BundleOptions{
@@ -74,6 +111,29 @@ func TestGenerateMarkdownIndexExplicitInput(t *testing.T) {
}
}
func TestGenerateMarkdownIndexWithCSSHref(t *testing.T) {
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
outputs, err := New().Generate(context.Background(), transform.Request{
SourceBackend: backend,
SourceBundle: sourceBundle,
Markdown: transform.MarkdownOptions{
Mode: transform.MarkdownModeIndex,
CssHref: "https://example.com/assets/report.css",
},
})
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
if got, want := outputs[0].Path, "index.html"; got != want {
t.Fatalf("path = %q, want %q", got, want)
}
if !strings.Contains(string(outputs[0].Data), `<link rel="stylesheet" href="https://example.com/assets/report.css">`) {
t.Fatalf("html = %q, want stylesheet link", outputs[0].Data)
}
}
func TestGenerateMarkdownIndexSelectsOnlyMarkdownFile(t *testing.T) {
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")

View File

@@ -1,10 +1,19 @@
package markdown
import "bytes"
import (
"bytes"
"html"
)
func wrapHTML(body []byte) []byte {
func wrapHTML(body []byte, cssHref string) []byte {
var buf bytes.Buffer
buf.WriteString("<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<title></title>\n</head>\n<body>\n")
buf.WriteString("<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n")
if cssHref != "" {
buf.WriteString("<link rel=\"stylesheet\" href=\"")
buf.WriteString(html.EscapeString(cssHref))
buf.WriteString("\">\n")
}
buf.WriteString("<title></title>\n</head>\n<body>\n")
buf.Write(body)
buf.WriteString("</body>\n</html>\n")
return buf.Bytes()

View File

@@ -25,6 +25,7 @@ type Request struct {
type MarkdownOptions struct {
Mode string
Input string
CssHref string
}
type Transformer interface {