From 484fda251478186e24064aec06d77c8e61237173 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 19 Jun 2026 16:50:40 +0000 Subject: [PATCH] Synchronize catalog documentation and examples --- docs/cli.md | 6 +- docs/config.md | 222 +++--------- docs/integrations/destination-state.md | 279 ++++---------- docs/integrations/markdown.md | 2 +- docs/internal/app.md | 12 +- docs/internal/config.md | 6 +- docs/internal/publish.md | 40 +- docs/internal/state.md | 52 ++- docs/operations.md | 90 ++--- docs/policy/architecture.md | 63 +--- docs/policy/development.md | 2 +- docs/troubleshooting.md | 341 +++--------------- ...{shared-root.yml => additive-workflow.yml} | 18 +- examples/archive-and-latest.yml | 2 + ...ciliation.yml => replacement-workflow.yml} | 13 +- internal/config/load_test.go | 19 +- 16 files changed, 296 insertions(+), 871 deletions(-) rename examples/{shared-root.yml => additive-workflow.yml} (59%) rename examples/{merge-reconciliation.yml => replacement-workflow.yml} (53%) diff --git a/docs/cli.md b/docs/cli.md index 6c2c701..cb4a153 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -86,7 +86,7 @@ distributor reconcile-state --config --pipeline --destination [ - `--config ` loads the pipeline configuration and is required. - `--pipeline ` selects the pipeline used to identify the destination root and is required. - `--destination ` selects the destination root and is required. -- `--all-owners` repairs missing managed output records for every owner in a shared-root state file. Without it, shared-root repair is scoped to the selected pipeline and destination owner. +- `--all-owners` repairs missing managed output records for every owner in the selected catalog state file. Without it, repair is scoped to the selected pipeline and destination owner. - `--dry-run` reports repairs without rewriting `.distributor.json`. - `--format text|json` selects human-readable or machine-readable output. @@ -229,7 +229,7 @@ go run ./cmd/distributor reconcile-state \ --destination local-archive ``` -Use `--all-owners` only for shared-root destination state when all owners inside the selected root should be repaired. +Use `--all-owners` only when every owner inside the selected catalog root should be repaired. ### Prune Managed Outputs @@ -292,7 +292,7 @@ Text output is optimized for direct operator use. JSON output is optimized for a - Usage errors and fatal setup errors exit non-zero and do not emit a JSON result document. - `run --format json` emits a JSON result for partial destination failures, sets `ok` to `false`, includes result details and errors, and exits non-zero. - Warnings are included in JSON output and are printed in text output when relevant. -- `run` summaries include separate `force_replace` counters. In JSON output, forced replacement actions use the stable action value `force_replace` and include the resolved `destination_path`. +- `run` summaries include `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and `fail_conflict` counters. Destination action records use the same stable action values and include the resolved `destination_path`. ## Diagnostics And Recovery diff --git a/docs/config.md b/docs/config.md index c3b408b..68a5791 100644 --- a/docs/config.md +++ b/docs/config.md @@ -2,11 +2,11 @@ Audience: administrators, operators, and advanced users who write YAML configuration for `distributor`. -This document is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operating procedures live in [Operations](operations.md), symptom-oriented recovery lives in [Troubleshooting](troubleshooting.md), and external contracts live under [Integrations](integrations/source-bundle.md). +This is the canonical user-facing configuration reference. CLI syntax lives in [CLI](cli.md), operations guidance lives in [Operations](operations.md), recovery guidance lives in [Troubleshooting](troubleshooting.md), and file-format contracts live under [Integrations](integrations/source-bundle.md). ## Config File Loading -`distributor run --config ` and `distributor serve --config ` load the YAML file at ``. If `--config` is omitted, both commands use: +`distributor run --config ` and `distributor serve --config ` load the YAML file at ``. If `--config` is omitted, commands use: ```text /usr/local/etc/distributor/config.yml @@ -21,7 +21,7 @@ Runtime backend support is command-specific: - `serve` uses `http_upload` sources through the HTTP upload API and publishes to configured `local`, `ssh`, and `s3` destinations. - `http_upload` is valid only as a source backend. -## Minimal Local Config +## Minimal Working Config ```yaml pipelines: @@ -35,9 +35,9 @@ pipelines: path: /srv/reports/archive ``` -This config publishes source files only. It uses default validation, destination path mapping, publish, state, reconciliation, retention, transfer, and HTTP server values. +This publishes source files only. It uses default validation, additive workflow, preserve-relative path mapping, source-only publish policy, disabled pruning, and default HTTP server values. -## Production-Oriented Local Config +## Production-Oriented Config ```yaml server: @@ -61,23 +61,20 @@ pipelines: - id: archive backend: local path: /srv/reports/archive + workflow: additive publish: source: true html: false path_mapping: mode: preserve_relative - reconciliation: - mode: replace - transfer: - on_destination_same: skip - on_destination_older: replace - on_destination_newer: skip - on_conflict: fail + retention: + prune: + enabled: false ``` ## HTTP Upload Source Config -HTTP upload sources are configured on pipelines and are served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML. +HTTP upload sources are configured on pipelines and served by `distributor serve`. Upload tokens are resolved from the process environment or `secrets.directory`; literal bearer tokens are not configured in YAML. ```yaml server: @@ -97,15 +94,13 @@ pipelines: - id: weather-daily source: backend: http_upload - staging_path: /var/spool/distributor/weather-daily - max_upload_size: 20MB destinations: - id: archive backend: local path: /srv/reports/archive ``` -`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to. One token may authorize multiple upload pipelines, and multiple tokens may authorize the same upload pipeline. +`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to. For `http_upload` sources, `staging_path` defaults to `/`. `max_upload_size` defaults to `server.http.max_upload_size`. @@ -136,7 +131,7 @@ See [Secrets](#secrets) for resolution rules. Each token has: -- `id`: required unique slug-like identifier for the token record. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`. +- `id`: required unique slug-like identifier for the token record. - `token_env`: required environment variable or secret-file name containing the bearer token value. - `allow_pipelines`: required non-empty list of configured pipeline ids whose source backend is `http_upload`. @@ -148,12 +143,12 @@ Token values must resolve to non-empty strings and must be unique across token r Each pipeline has: -- `id`: required unique slug-like identifier. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`. +- `id`: required unique slug-like identifier. - `source`: required source backend config. - `validation`: optional validation policy. - `destinations`: required non-empty destination list. -Pipeline ids must be unique across the config. +Slug-like identifiers must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`. ## Backend Reference @@ -171,7 +166,7 @@ path: /srv/distributor/archive ### SSH/SFTP Backend -SSH backends use native SFTP and can be used as sources and destinations. Adapter protocol behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md). +SSH backends use native SFTP and can be used as sources and destinations. Adapter behavior is documented in [SSH/SFTP Integration](integrations/ssh-sftp.md). ```yaml backend: ssh @@ -193,17 +188,11 @@ host_key_policy: strict - `known_hosts`: optional OpenSSH `known_hosts` path. - `host_key_policy`: optional host key policy. Default: `accept-new`. -Accepted host key policy values: - -- `strict` or boolean `true`: require a matching known host key. -- `accept-new`: accept and persist a new host key, but reject changed known keys. -- `off` or boolean `false`: disable host key checking. - -Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML. +Accepted host key policy values are `strict` or boolean `true`, `accept-new`, and `off` or boolean `false`. Authentication uses SSH agent identities when `SSH_AUTH_SOCK` is available, then `ssh_key_file` when configured. Password authentication is not configured in YAML. ### S3-Compatible Backend -S3 backends can be used as sources and destinations. Adapter protocol behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md). +S3 backends can be used as sources and destinations. Adapter behavior is documented in [S3-Compatible Storage Integration](integrations/s3.md). ```yaml backend: s3 @@ -255,42 +244,48 @@ Source bundle digest mismatches fail validation before destination writes occur. ## Destination Fields -Each destination embeds a backend config at the destination level and may also configure publishing, transforms, path mapping, links, state, reconciliation, retention, and transfer behavior. +Each destination embeds a backend config at the destination level and may also configure workflow, publishing, transforms, path mapping, links, and retention. ```yaml destinations: - id: archive backend: local path: /srv/reports/archive + workflow: additive publish: source: true html: false path_mapping: mode: preserve_relative - state: - mode: single_owner - reconciliation: - mode: replace - transfer: - on_destination_same: skip - on_destination_older: replace - on_destination_newer: skip - on_conflict: fail ``` - `id`: required unique slug-like identifier within the pipeline. - Backend fields: required according to the selected destination backend. +- `workflow`: optional catalog update workflow. Default: `additive`. - `publish`: optional publish policy. Default: source-only publication. - `transform`: required only when publishing generated HTML. - `path_mapping`: optional destination path mapping policy. - `links`: optional public URL metadata policy. -- `state`: optional destination state ownership policy. -- `reconciliation`: optional managed-output reconciliation policy. - `retention`: optional managed-output retention policy. -- `transfer`: optional destination comparison action policy. Destination ids must be unique within a pipeline. +Legacy destination policy keys `state`, `reconciliation`, `takeover`, and `transfer` are not accepted config fields. YAML files containing those keys fail during config loading. + +## Destination Workflow + +```yaml +workflow: additive +``` + +- `workflow`: optional. Accepted values are `additive` and `replacement`; default is `additive`. + +`additive` writes planned outputs into the catalog and retains unrelated catalog-managed outputs in the same destination bundle path. Existing catalog records for planned paths are replaced by the current publication. A planned path that exists in storage but is not recorded in valid catalog state fails as unmanaged content unless `run --force` selects `force_replace`. + +`replacement` writes planned outputs for the current pipeline and destination, and removes catalog outputs owned by the same pipeline and destination when those outputs are omitted from the new plan. Outputs owned by other pipeline/destination pairs remain catalog-managed. Replacement workflow is normal managed behavior and does not require `--force`. + +Use `replacement` for stable latest-style destinations where the current owner should publish exactly the currently planned output set. Use `additive` when a destination root intentionally accumulates outputs over time or receives disjoint outputs from multiple configured destinations. + ## Publish And Transform Policy ### Source-Only Publication @@ -365,113 +360,7 @@ Primary URL policies: - `html`: use the first generated HTML output. - `source`: use the first copied source output. -If no output matches the primary policy, per-output URLs may still be recorded and the top-level primary URL is omitted. - -## Destination State Policy - -```yaml -state: - mode: single_owner -``` - -- `state.mode`: optional. Accepted values are `single_owner` and `shared_root`; default is `single_owner`. - -`single_owner` state records one pipeline/destination owner for each destination bundle path and is the default state mode. - -`shared_root` records multiple pipeline/destination owners in one destination root. Publish execution preserves unrelated owners, rejects path ownership conflicts, and writes shared-root destination state. - -Use `shared_root` when multiple configured destinations intentionally write disjoint output paths into the same backend root: - -```yaml -pipelines: - - id: reports-source - source: - backend: local - path: /var/spool/distributor/reports - destinations: - - id: shared-root - backend: local - path: /srv/reports/shared - state: - mode: shared_root - publish: - source: true - html: false - - id: reports-html - source: - backend: local - path: /var/spool/distributor/reports - destinations: - - id: shared-root - backend: local - path: /srv/reports/shared - state: - mode: shared_root - publish: - source: false - html: true - transform: - markdown_to_html: - enabled: true - mode: sidecar -``` - -Every output path in a shared root belongs to exactly one `pipeline_id` and `destination_id`. A different owner planning the same path fails as a conflict unless `takeover.mode` allows the managed output path to move to the current owner. - -## Takeover Policy - -```yaml -takeover: - mode: same_pipeline -``` - -- `takeover.mode`: optional. Accepted values are `same_pipeline`, `same_source`, `any_managed`, and `never`; default is `same_pipeline`. - -Takeover controls when a destination may normally replace valid distributor-managed state whose pipeline, destination, source identity, or shared-root output owner differs from the current publication. - -- `same_pipeline`: replace managed state owned by the same pipeline. The previous destination id and source id may differ. -- `same_source`: replace managed state only when the existing source manifest id matches the current source id. -- `any_managed`: replace any valid distributor-managed state at the selected destination bundle path. -- `never`: do not replace identity or source conflicts without the explicit forced replacement workflow. - -For shared-root state, `same_pipeline` permits taking over output paths owned by another destination in the same pipeline, `same_source` permits taking over output paths whose owner records the same source manifest id, and `any_managed` permits taking over output paths owned by any valid shared-root owner. Unrelated owner records and non-conflicting outputs remain managed by their existing owners. - -Takeover does not apply to unmanaged content, invalid destination state, same-created digest conflicts, or same-source destination-newer comparisons. - -## Reconciliation Policy - -```yaml -reconciliation: - mode: replace -``` - -- `reconciliation.mode`: optional. Accepted values are `replace` and `merge`; default is `replace`. - -Reconciliation controls how a destination with older managed state is updated after transfer policy selects `replace_older`. - -`replace` deletes the prior managed outputs recorded in `.distributor.json`, deletes the state file, verifies the destination bundle path is empty, then writes only the newly planned outputs and a new state file. This is the default and is appropriate when each publication should exactly match the current publish and transform policy. - -`merge` keeps prior managed outputs that are not produced by the new plan. Planned paths already recorded in existing state may be overwritten; planned paths that already exist in storage but are not recorded as managed fail as unmanaged collisions. The resulting state file records the cumulative managed output set. - -Example merge destination: - -```yaml -destinations: - - id: static-site - backend: local - path: /srv/reports/site - publish: - source: false - html: true - transform: - markdown_to_html: - enabled: true - mode: sidecar - reconciliation: - mode: merge -``` - -`links.primary_url` is selected from the newly planned outputs for the current run. Retained outputs keep their prior output metadata and timestamps. +If no output matches the primary policy, per-output URLs may still be recorded and no primary URL is reported for the run. ## Retention Policy @@ -489,28 +378,9 @@ retention: When `retention.prune.enabled` is `true`, at least one of `older_than` or `keep_latest` is required. `older_than` must be greater than zero, and `keep_latest` must be zero or greater. -Pruning uses managed output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs. +Pruning uses catalog output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs. -The `prune` command is owner-scoped for shared-root state. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`. - -## Transfer Policy - -```yaml -transfer: - on_destination_same: skip - on_destination_older: replace - on_destination_newer: skip - on_conflict: fail -``` - -Transfer fields and accepted values: - -- `transfer.on_destination_same`: `skip` or `fail`. Default: `skip`. -- `transfer.on_destination_older`: `replace` or `fail`. Default: `replace`. -- `transfer.on_destination_newer`: `skip`, `replace`, or `fail`. Default: `skip`. -- `transfer.on_conflict`: `fail` or `replace`. Default: `fail`. - -`replace` for `on_destination_newer` and `on_conflict` applies only to valid distributor-managed destination state. Unmanaged destination content and invalid destination state still require the explicit `run --force` workflow where supported. +The `prune` command is scoped to the selected pipeline and destination owner. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`. ## Size And Duration Values @@ -543,18 +413,12 @@ Defaults are applied after YAML decoding and before validation: - S3 `force_path_style: true` - `http_upload` source `staging_path: /` - `http_upload` source `max_upload_size: server.http.max_upload_size` +- `workflow: additive` - `publish.source: true` and `publish.html: false` - `transform.markdown_to_html.mode: sidecar` when a Markdown transform block is present and mode is omitted - `path_mapping.mode: preserve_relative` - `links.primary: auto` when a `links` block is present and `primary` is omitted -- `state.mode: single_owner` -- `reconciliation.mode: replace` -- `takeover.mode: same_pipeline` - `retention.prune.enabled: false` -- `transfer.on_destination_same: skip` -- `transfer.on_destination_older: replace` -- `transfer.on_destination_newer: skip` -- `transfer.on_conflict: fail` ## Secrets @@ -577,7 +441,7 @@ Fields resolved through this resolver: ## Maintained Examples -Maintained examples live under [examples](../examples/). Config tests load every file listed here. +Maintained examples live under [examples](../examples/). Config tests load these YAML files. Local examples: @@ -587,8 +451,8 @@ Local examples: - `local-index.yml`: local `index.html` publication. - `fan-out.yml`: local fan-out publication to source and HTML destinations. - `archive-and-latest.yml`: local archive plus fixed latest publication. -- `merge-reconciliation.yml`: local HTML publication using merge reconciliation. -- `shared-root.yml`: two local pipelines publishing disjoint outputs into one shared destination root. +- `additive-workflow.yml`: two destinations publishing disjoint outputs into one catalog-managed root. +- `replacement-workflow.yml`: fixed-path replacement workflow for a stable latest-style output set. - `http-upload-local.yml`: local HTTP upload server config; requires `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` in the process environment or as a secret-file name before running `serve`. Environment-gated remote examples: diff --git a/docs/integrations/destination-state.md b/docs/integrations/destination-state.md index e392c4b..c667796 100644 --- a/docs/integrations/destination-state.md +++ b/docs/integrations/destination-state.md @@ -2,197 +2,37 @@ Audience: operators, integrators, and maintainers who inspect or reason about destination `.distributor.json` files. -Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for comparison, skip, replacement, and recovery decisions. +Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for catalog planning, managed replacement, retention pruning, repair, and recovery. -## Single-Owner State Schema +## Catalog State Schema -State written by `run` for `state.mode: single_owner` uses schema version `2`. +Publish execution writes catalog state with `schema_version` `4`. ```json { - "schema_version": 2, - "distributor_version": "dev", - "pipeline_id": "reports", - "destination_id": "archive", - "published_at": "2026-06-04T12:00:00Z", - "created_at": "2026-06-04T12:00:00Z", - "updated_at": "2026-06-04T12:00:00Z", - "state": { - "mode": "single_owner" - }, - "reconciliation": { - "mode": "replace" - }, - "source": { - "manifest": { - "schema_version": 1, - "id": "reports.example.2026-06-04", - "digest": "sha256:...", - "created": "2026-06-04T11:55:00Z", - "files": [ - {"path": "report.md", "sha256": "sha256:...", "size": 1234} - ] - } - }, - "links": { - "primary_url": "https://reports.example.com/archive/report.html" - }, - "outputs": [ - { - "path": "report.html", - "kind": "generated", - "source_path": "report.md", - "transform": "markdown_to_html", - "url": "https://reports.example.com/archive/report.html", - "sha256": "sha256:...", - "size": 2345, - "created_at": "2026-06-04T12:00:00Z", - "updated_at": "2026-06-04T12:00:00Z" - } - ] -} -``` - -Required fields: - -- `schema_version`: must be `2` for newly written state. -- `pipeline_id`: configured pipeline id that wrote the state. -- `destination_id`: configured destination id that wrote the state. -- `published_at`: RFC3339 publication timestamp. -- `created_at`: RFC3339 timestamp for when this destination state record was first created. -- `updated_at`: RFC3339 timestamp for the latest state update. -- `state.mode`: must be `single_owner`. -- `reconciliation.mode`: `replace` or `merge`. -- `source.manifest`: embedded source bundle manifest. -- `outputs`: output records array; empty is allowed, but the field is required. - -Optional fields: - -- `distributor_version`: application version string when available. -- `links.primary_url`: absolute HTTP or HTTPS URL selected by destination link policy. - -## Output Records - -Each output record has: - -- `path`: destination-relative output path. -- `kind`: `source` or `generated`. -- `source_path`: source manifest path used for the output. -- `transform`: required for `generated` outputs; omitted for copied source outputs. -- `url`: optional absolute HTTP or HTTPS URL for the output. -- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes. -- `size`: output byte size, zero or greater. -- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed. -- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state. - -Output paths must be unique and use clean relative slash-separated path rules. - -For replacement updates, newly planned outputs are written into state. For merge updates, retained output records preserve both timestamps, overwritten managed output records preserve `created_at` and receive a new `updated_at`, and new output records receive the current publication time for both fields. - -## Reconciliation Semantics - -Destination reconciliation applies when destination state is older than the source and transfer policy permits replacement. - -- `replace`: for single-owner state, delete managed output paths recorded in `outputs` plus `.distributor.json`, require the destination bundle path to be empty afterward, write the newly planned outputs, and write state whose `outputs` are exactly that new planned set. For shared-root state, delete only the current owner's omitted outputs and preserve unrelated owners. -- `merge`: retain managed output paths omitted from the new plan, overwrite planned paths only when they are already recorded in existing state, fail when a newly planned path exists in storage but is not recorded as managed, and write state whose `outputs` are the cumulative managed set. - -Takeover replacement applies when valid managed state has an identity, source, or shared-root output-owner mismatch and destination `takeover.mode` permits the current publication to take ownership. It uses bounded managed replacement behavior. For shared-root state, only taken-over output records and the current owner records are rewritten; unrelated owner records and non-conflicting outputs remain managed by their existing owners. - -Top-level `links.primary_url` is selected from the newly planned outputs for the current publication. Retained outputs keep their existing per-output URL metadata. - -If a same-source merge publication fails after writing outputs, cleanup removes only newly created outputs from that failed attempt. Previously managed overwritten paths remain managed and are not removed by failed-attempt cleanup. Takeover replacement does not retain omitted outputs through merge reconciliation. - -## Comparison Semantics - -`distributor` compares the current source manifest to destination state before writing: - -- No state and no content: publish new outputs. -- No state and existing content: treat the destination as unmanaged. -- Shared-root state without the current owner: publish new outputs for that owner if planned paths do not collide with unmanaged content or with other owners that `takeover.mode` does not permit. -- Matching embedded source manifest: skip. -- Same source id with older `created`: replace if policy allows. -- Same source id with newer `created`: skip by default, or replace when `transfer.on_destination_newer: replace` is configured. -- Same source id and same `created` with different digest: conflict by default, or replace when `transfer.on_conflict: replace` is configured. -- Different source id, pipeline id, or destination id in single-owner state: conflict unless `takeover.mode` permits managed ownership transfer or `transfer.on_conflict: replace` is configured. -- Shared-root output path owned by a different owner: conflict unless `takeover.mode` permits managed ownership transfer or `transfer.on_conflict: replace` is configured. -- Invalid state JSON or invalid state fields: conflict. - -Normal managed replacement deletes only managed output paths recorded in `.distributor.json`. Forced replacement deletes the bounded destination bundle path, then writes planned outputs and schema version `4` catalog state. - -## State Repair Semantics - -`distributor reconcile-state` can remove managed output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`. - -For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. The command checks output paths recorded in `outputs`, reports missing managed outputs, reports unmanaged entries under the destination root, and removes missing output records from valid state unless `--dry-run` is set. - -For shared-root state, repair is scoped to the selected owner by default. With `--all-owners`, it checks and repairs missing output records for every owner in the selected shared-root state file. - -State repair does not validate output digests, delete destination files, adopt unmanaged entries, or rewrite invalid or mismatched state. - -## Prune Semantics - -`distributor prune` can delete managed outputs selected by the configured destination retention policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`. - -For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners. - -Prune planning uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`. - -## Compatibility - -`distributor` can read schema version `1` destination state for compatibility. When v1 state is read, it is treated as single-owner state with `reconciliation.mode: replace`. Missing top-level `created_at` and `updated_at` are inferred from `published_at`, and missing per-output timestamps are also inferred from `published_at`. - -Newly written single-owner destination state from publish execution uses schema version `2`. - -## Shared-Root State Schema - -State written by `run` for `state.mode: shared_root` uses schema version `3`. - -```json -{ - "schema_version": 3, + "schema_version": 4, "distributor_version": "dev", "created_at": "2026-06-04T12:00:00Z", "updated_at": "2026-06-04T12:10:00Z", "state": { - "mode": "shared_root" + "mode": "catalog" }, - "owners": [ - { - "pipeline_id": "reports", - "destination_id": "archive", - "reconciliation": { - "mode": "merge" - }, - "source": { - "manifest": { - "schema_version": 1, - "id": "reports.example.2026-06-04", - "digest": "sha256:...", - "created": "2026-06-04T11:55:00Z", - "files": [ - {"path": "report.md", "sha256": "sha256:...", "size": 1234} - ] - } - }, - "links": { - "primary_url": "https://reports.example.com/archive/report.html" - } - } - ], "outputs": [ { "path": "report.html", + "pipeline_id": "reports", + "destination_id": "static-site", + "source": { + "id": "reports.example.2026-06-04", + "digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "created": "2026-06-04T11:55:00Z" + }, "kind": "generated", "source_path": "report.md", "transform": "markdown_to_html", "url": "https://reports.example.com/archive/report.html", - "sha256": "sha256:...", + "sha256": "sha256:abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd", "size": 2345, - "pipeline_id": "reports", - "destination_id": "archive", - "source_id": "reports.example.2026-06-04", - "source_digest": "sha256:...", - "source_created": "2026-06-04T11:55:00Z", "created_at": "2026-06-04T12:00:00Z", "updated_at": "2026-06-04T12:10:00Z" } @@ -200,55 +40,88 @@ State written by `run` for `state.mode: shared_root` uses schema version `3`. } ``` -Shared-root required fields: +Top-level fields: -- `schema_version`: must be `3`. -- `created_at`: RFC3339 timestamp for when this shared-root state record was first created. -- `updated_at`: RFC3339 timestamp for the latest shared-root state update. -- `state.mode`: must be `shared_root`. -- `owners`: owner records keyed by `pipeline_id` and `destination_id`; each owner records its latest source manifest and reconciliation mode. -- `outputs`: output records for every managed path under the shared destination root. +- `schema_version`: required value `4`. +- `distributor_version`: optional application version string. +- `created_at`: RFC3339 timestamp for when this catalog record was first created. +- `updated_at`: RFC3339 timestamp for the latest catalog update. +- `state.mode`: required value `catalog`. +- `outputs`: required array of catalog output records. Empty is valid. -Shared-root optional fields: +## Output Records -- `distributor_version`: application version string when available. -- `owners[].links.primary_url`: absolute HTTP or HTTPS URL selected by that owner's destination link policy. +Each output record has: -Shared-root output records carry the same `path`, `kind`, `source_path`, `transform`, `url`, `sha256`, `size`, `created_at`, and `updated_at` fields as single-owner outputs. They also include the owner `pipeline_id` and `destination_id`, plus compact source identity fields `source_id`, `source_digest`, and `source_created`. +- `path`: destination-bundle-relative output path. +- `pipeline_id`: configured pipeline id that manages the output path. +- `destination_id`: configured destination id that manages the output path. +- `source`: compact source identity for the output. +- `kind`: `source` or `generated`. +- `source_path`: generated outputs only; source manifest path used to derive the output. +- `transform`: generated outputs only; transform id, currently `markdown_to_html`. +- `url`: optional absolute HTTP or HTTPS URL for the output. +- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes. +- `size`: output byte size, zero or greater. +- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed. +- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state. -## Shared-Root Ownership +Output paths must be unique and use clean relative slash-separated path rules. `pipeline_id` and `destination_id` must be slug-like identifiers. -Shared-root state is owner-scoped by `pipeline_id` and `destination_id`. +For copied source outputs, `source_path` and `transform` are omitted. For generated outputs, both fields are required. -- One output path may be owned by only one owner. -- The same owner may overwrite its own managed paths. -- A different owner planning an already owned path fails as a conflict. -- A planned path that exists in storage but is not recorded in state fails as unmanaged content unless forced replacement is explicitly selected. +## Source Identity -When an owner publishes, unrelated owner records and output records are preserved. The publishing owner's record is updated with the latest source manifest, reconciliation mode, and latest primary URL when present. +Each output records the source identity that produced it: -## Shared-Root Timestamps +- `source.id`: source manifest id. +- `source.digest`: source manifest digest. +- `source.created`: source manifest creation timestamp. -For shared-root state: +The full source manifest is not embedded in catalog state. The source bundle's `manifest.json` remains the producer contract, and `.distributor.json` records only the source identity needed for catalog ownership and later maintenance workflows. -- top-level `created_at` remains the original shared-root state creation time; -- top-level `updated_at` changes after a successful state write; -- output `created_at` remains stable for an existing managed path; -- output `updated_at` changes only when that path is rewritten; -- newly managed output paths receive the publication time for both output timestamps. +## Workflow Semantics -## Shared-Root Migration +Destination `workflow` is runtime configuration and is not persisted in `.distributor.json`. -If `state.mode: shared_root` is configured and existing state is a compatible single-owner `.distributor.json` for the same pipeline id and destination id, the next successful publish writes schema version `3` shared-root state for that owner. +`workflow: additive` writes planned outputs and retains unrelated catalog-managed outputs. If a planned path already has a catalog output record, the current publication replaces that record and overwrites the file. If a planned path exists in storage but is not recorded in valid catalog state, planning fails as unmanaged unless `run --force` selects `force_replace`. -If `state.mode: shared_root` is configured and existing single-owner state belongs to a different pipeline or destination, publish fails as a conflict. `distributor` does not implicitly convert unrelated single-owner state or take over unmanaged files during shared-root migration. +`workflow: replacement` writes planned outputs for the current pipeline and destination and removes omitted outputs owned by that same pipeline and destination. Outputs owned by other pipeline/destination pairs remain catalog-managed. This is normal managed replacement and does not require `--force`. + +`force_replace` is an explicit per-run recovery path. It deletes only the resolved destination bundle path, then writes planned outputs and fresh catalog state. It can replace unmanaged content, planned unmanaged path collisions, invalid destination state, and unsupported future destination state after dry-run review. + +## Publish Planning Outcomes + +Current run reports use these destination action labels: + +- `publish_new`: no valid state exists and the destination bundle path is empty. +- `upsert_additive`: valid catalog state exists and additive workflow will write the planned outputs. +- `replace_catalog`: valid catalog state exists and replacement workflow will write the planned outputs and remove omitted outputs for the current owner. +- `skip_same`: no-op action value in the run output vocabulary. +- `force_replace`: explicit bounded destructive replacement selected by `--force`. +- `fail_unmanaged`: unmanaged destination content prevents publication. +- `fail_conflict`: invalid state or unsupported state prevents publication without explicit force. + +Schema versions older than `4` are superseded legacy state for publish planning. Normal catalog planning may publish over superseded legacy state according to the configured workflow, while invalid state and unsupported future schema versions fail unless `--force` is explicitly selected. + +## Repair Semantics + +`distributor reconcile-state` removes catalog output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`. + +By default, repair is scoped to output records whose `pipeline_id` and `destination_id` match the selected pipeline and destination. With `--all-owners`, it checks every catalog output record in the selected root. + +The command reports missing managed outputs and unmanaged storage entries. Without `--dry-run`, it removes missing managed output records from valid catalog state and rewrites `.distributor.json`. It does not delete destination files, adopt unmanaged entries, validate output digests, or rewrite invalid state. + +## Prune Semantics + +`distributor prune` deletes catalog output paths selected by the configured destination `retention.prune` policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`. + +Prune planning is scoped to output records whose `pipeline_id` and `destination_id` match the selected pipeline and destination. It uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid catalog state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`. ## Boundaries Destination state is internal managed state written by `distributor`. Operators may inspect it during recovery, but normal workflows should not edit it by hand. Source `manifest.json` is not copied as destination state. -## Tests - Before changing this contract, inspect and run: ```sh diff --git a/docs/integrations/markdown.md b/docs/integrations/markdown.md index bdac338..8b9e7a2 100644 --- a/docs/integrations/markdown.md +++ b/docs/integrations/markdown.md @@ -50,7 +50,7 @@ Destination state stores generated outputs with `kind: generated`, `source_path` ## Boundaries -Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose transfer policy. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata. +Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select destination actions, or choose catalog workflow behavior. Publish planning decides whether generated HTML is selected for a destination and destination state records the generated output metadata. ## Tests diff --git a/docs/internal/app.md b/docs/internal/app.md index 827d890..9cd8793 100644 --- a/docs/internal/app.md +++ b/docs/internal/app.md @@ -20,7 +20,7 @@ User-facing command parsing stays in `internal/cli`, including `reconcile-state` ## Config Fields Used -The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, takeover policy, retention policy, and transfer policy. +The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, workflow, publish policy, transform policy, path mapping, links, and retention policy. Config fields are validated and defaulted by `internal/config` before app workflows use them. @@ -32,11 +32,11 @@ The app layer registers default transforms, including Markdown-to-HTML, and supp ## State And Manifest Behavior -Run workflows discover and validate source bundles through `internal/bundle`. Destination state actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results. Run summaries count older-state, conflict, newer-state, takeover, and explicit forced replacements separately. +Run workflows discover and validate source bundles through `internal/bundle`. Destination catalog actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results. Run summaries count `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, and `fail_conflict` separately. -Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing managed output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from state and rewrites valid state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid or mismatched state. +Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing catalog output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from valid catalog state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid state. -Prune planning consumes a parsed destination state document and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state. +Prune planning consumes parsed catalog state and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state. Prune execution loads one configured pipeline/destination selector, opens that destination root, parses the root `.distributor.json`, and builds a plan from the destination retention policy. Dry-run returns the same planned and preserved managed output records without deleting files or rewriting state. Apply mode deletes only planned managed output paths, never unmanaged files or `.distributor.json`, then removes confirmed deleted records from state and updates the state timestamp. If a delete fails after earlier deletes succeeded, it rewrites state only for the confirmed deletions and preserves records for the failed and unattempted outputs so a retry remains accurate. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. @@ -54,9 +54,9 @@ HTTP upload status is in memory. Accepted jobs move through accepted, queued, ru Runtime setup fails for config load, config validation, secret loading, or credential resolution errors. Source setup failures stop the affected run before destination planning. Destination open, planning, execution, and notification failures are recorded as destination failures where a partial result exists. -Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root all-owner repair still uses the selected destination to identify the root, then applies repair across owners inside that root. Invalid, unreadable, or ambiguous state fails before any rewrite. +Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Catalog repair is scoped to that owner unless `--all-owners` is set. Invalid or unreadable state fails before any rewrite. -Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root pruning is scoped to the selected owner and preserves unrelated owners. Invalid, unreadable, or ambiguous state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output. +Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Pruning is scoped to the selected owner and preserves unrelated owners. Invalid or unreadable state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output. HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Upload requests can fail during authentication, idempotency-key validation, content-type validation, idempotency conflict checks, queue admission, archive staging, source validation, or later publish execution. diff --git a/docs/internal/config.md b/docs/internal/config.md index a722a7c..7db0c2b 100644 --- a/docs/internal/config.md +++ b/docs/internal/config.md @@ -18,7 +18,7 @@ The canonical user-facing config reference is `docs/config.md`. ## Config Fields Used -The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, takeover policy, retention policy, and transfer policy. +The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, destination workflow, publish policy, transform policy, path mapping, links, and retention policy. ## Adapters Used @@ -26,11 +26,11 @@ No external storage adapters are used directly. The package exposes normalized c ## State And Manifest Behavior -The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as publish/transform combinations, links, state policy, reconciliation policy, takeover policy, retention policy, transfer policy, backend roots, S3 prefix shape, and HTTP upload source settings. +The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as workflow, publish/transform combinations, links, retention policy, backend roots, S3 prefix shape, and HTTP upload source settings. Legacy destination policy YAML fields are rejected by strict decoding because they are not user-visible config fields. ## Skip And Resume Behavior -The package has no runtime skip or resume behavior. It provides transfer policy values that publish planning later applies to destination comparison outcomes. +The package has no runtime skip or resume behavior. Publish planning later applies workflow values and per-run force options to destination catalog outcomes. ## Failure Behavior diff --git a/docs/internal/publish.md b/docs/internal/publish.md index bfb1627..d8192f3 100644 --- a/docs/internal/publish.md +++ b/docs/internal/publish.md @@ -4,13 +4,13 @@ Audience: developers and LLM coding agents changing `internal/publish`. ## Purpose -`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns destination comparison mapping, output selection, URL planning, managed cleanup selection, replacement safety, and destination state projection. +`internal/publish` plans and executes publication for one validated source bundle and one destination bundle path. It owns output selection, URL planning, catalog action selection, managed cleanup selection, forced replacement safety, and destination state projection. ## Inputs And Outputs -Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, state policy, reconciliation policy, takeover policy, transformer resolver, transfer policy, distributor version, and force flag. +Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, workflow, transformer resolver, distributor version, force flag, and request time. -Output from planning is a `Plan` with action, reason, destination identity, selected outputs, state mode, owner scope, reconciliation mode, takeover mode, optional existing single-owner or shared-root state, optional primary URL, and force metadata. Shared-root plans also expose other-owner outputs to preserve, taken-over outputs, current-owner outputs retained by merge, current-owner outputs deleted by replace or takeover, and current-owner outputs to write. Execution writes selected source outputs, generated outputs, and `.distributor.json` for executable publish or replacement actions. +Output from planning is a `Plan` with action, reason, destination identity, selected outputs, workflow, owner scope, optional existing catalog state, optional superseded legacy marker, optional primary URL, catalog outputs to write, catalog outputs to retain, catalog outputs to delete, force metadata, and clear-root metadata. Execution writes selected source outputs, generated outputs, and schema version `4` `.distributor.json` for executable catalog actions. ## Boundaries @@ -20,7 +20,7 @@ External destination state semantics are documented in `docs/integrations/destin ## Config Fields Used -The package consumes already-defaulted config values for destination `publish`, `transform`, `links`, `state`, `reconciliation`, `takeover`, `transfer`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency. +The package consumes already-defaulted config values for destination `workflow`, `publish`, `transform`, `links`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency. ## Adapters Used @@ -28,27 +28,34 @@ The package depends on `internal/storage.Backend` for source and destination IO, ## State And Manifest Behavior -Planning inspects destination state through `internal/state`, compares it with the source manifest, and maps comparison outcomes plus transfer and takeover policy into actions. Catalog planning uses `publish_new`, `upsert_additive`, `replace_catalog`, `force_replace`, `fail_conflict`, or `fail_unmanaged`; legacy action labels remain only for older planning and removal work scheduled separately. +Planning inspects destination state through `internal/state` and maps catalog conditions into actions: -Single-owner destinations compare the whole destination state against the configured pipeline and destination ids. Valid managed identity and source conflicts can become `replace_takeover` when `takeover.mode` allows them. Shared-root destinations compare only the current owner scope, keyed by pipeline id and destination id. An absent shared-root owner is publishable for that owner unless a planned output collides with unmanaged storage content. Planned writes to a path owned by another shared-root owner become `replace_takeover` when `takeover.mode` allows that managed output path to move to the current owner. +- `publish_new`: no valid state exists and the destination bundle path is empty. +- `upsert_additive`: additive workflow writes planned outputs and retains unrelated catalog outputs. +- `replace_catalog`: replacement workflow writes planned outputs and deletes omitted outputs for the current owner. +- `force_replace`: explicit bounded replacement selected by `Force`. +- `fail_unmanaged`: unmanaged destination content blocks publication. +- `fail_conflict`: invalid or unsupported state blocks publication. -Execution writes destination state after selected outputs are written. Destination state includes copied source output metadata, generated output metadata, output timestamps, embedded source manifest, reconciliation metadata, link metadata when configured, pipeline id, destination id, and publication timestamps. +Superseded legacy state is identified by schema number and planned through catalog output projection. A successful publish writes schema version `4` catalog state. -## Skip And Resume Behavior +Execution writes destination state after selected outputs are written. Catalog output records include owner identity, compact source identity, copied source output metadata, generated output metadata, output timestamps, and optional URL metadata. -`skip_same` and `skip_destination_newer` execute as no-ops. Replacement-mode single-owner updates remove managed output paths from existing state plus `.distributor.json`, verify the destination is empty, and write state whose outputs are exactly the new plan. `replace_conflict` uses managed replacement mechanics and does not retain omitted outputs through merge reconciliation. `replace_newer` follows the same managed replacement and merge-retention rules as `replace_older`. `replace_takeover` uses managed replacement mechanics and does not retain omitted outputs through merge reconciliation. Replacement-mode shared-root updates remove only current-owner omitted outputs and preserve unrelated owners. Shared-root takeover rewrites only the taken-over output records and current owner records. Merge-mode same-source updates retain omitted managed outputs, overwrite only paths already recorded as managed, reject unmanaged destination path collisions, and write cumulative output state. Failed writes trigger cleanup where practical; same-source merge cleanup removes only newly created outputs from the failed attempt. +## Workflow Behavior -Shared-root execution writes schema version `3` state. It preserves unrelated owner records and outputs, updates only the publishing owner metadata, preserves root `created_at`, and updates root `updated_at` after successful state writes. Compatible single-owner state for the same pipeline and destination is converted to shared-root state on successful publish. +Additive workflow computes a write set for the planned outputs and preserves catalog outputs for unplanned paths. Existing catalog records for planned paths are replaced by the current owner and source identity. -Forced replacement is explicit per request and deletes the bounded destination bundle path before writing planned outputs and schema version `4` catalog state. Catalog planning selects `force_replace` only when `Force` is true and normal planning would otherwise fail for a non-empty no-state destination, a planned path collision with unmanaged storage content, invalid destination state, or unsupported future destination state. Valid catalog-managed additive upserts and replacement workflow publishes are normal managed actions; they are not forced replacement actions. +Replacement workflow computes a write set for the planned outputs, preserves other-owner outputs, and deletes omitted outputs owned by the current pipeline and destination. It does not need `Force`. + +Forced replacement is explicit per request. It deletes the bounded destination bundle path before writing planned outputs and schema version `4` catalog state. Catalog planning selects `force_replace` only when `Force` is true and normal planning would otherwise fail for a non-empty no-state destination, a planned path collision with unmanaged storage content, invalid destination state, or unsupported future destination state. Retention pruning is not part of publish execution and does not run automatically after a successful publish. The app-level prune workflow uses destination state after publication to select managed outputs for deletion. ## Failure Behavior -Planning fails for incomplete requests, invalid publish/transform policy, invalid state mode, invalid reconciliation mode, output path collisions, invalid destination state, unmanaged destination content without force, shared-root owner path conflicts not allowed by `takeover.mode`, conflict outcomes not allowed by transfer policy, unresolved transforms, invalid Markdown output selection, and invalid link URL planning. +Planning fails for incomplete requests, invalid publish/transform policy, invalid workflow, output path collisions, unresolved transforms, invalid Markdown output selection, invalid link URL planning, invalid destination state without force, unmanaged destination content without force, and unsupported future state without force. -Execution fails on delete, read, transform output, unmanaged merge path collision, shared-root ownership conflict, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable publish or replacement actions. +Execution fails on delete, read, transform output, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable catalog publish or replacement actions. ## Tests To Inspect @@ -62,11 +69,8 @@ Execution fails on delete, read, transform output, unmanaged merge path collisio - Planning is deterministic for the same request and destination state. - Destination bundle paths are caller-supplied and backend-root-relative. - URL generation uses URL path semantics and never infers public URLs from backend config. -- Replacement reconciliation deletes only managed paths recorded in existing state plus `.distributor.json` for single-owner state, and only current-owner omitted outputs for shared-root state. -- Merge reconciliation never adopts unmanaged content. -- Merge state output records are cumulative for the single owner. -- Shared-root planning is owner-scoped, preserves unrelated owner outputs, and records taken-over managed output paths separately from unrelated owners. -- Shared-root execution writes owner-scoped changes without deleting unrelated owners. +- Additive workflow never adopts unmanaged storage content. +- Replacement workflow deletes only catalog outputs owned by the current pipeline and destination unless force replacement clears the bounded destination bundle path. - Forced replacement deletes only within the supplied destination bundle path and then writes catalog state using the same output projection as normal catalog publish planning. - Destination state is written after selected outputs are written. - Transform resolution stays behind a caller-supplied interface. diff --git a/docs/internal/state.md b/docs/internal/state.md index 8099dce..cba24b0 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -4,21 +4,21 @@ Audience: developers and LLM coding agents changing `internal/state`. ## Purpose -`internal/state` parses, validates, serializes, and compares `.distributor.json` destination state. +`internal/state` parses, validates, serializes, and transforms `.distributor.json` destination catalog state. ## Inputs And Outputs -Inputs are destination state JSON, constructed state values, current source manifest, pipeline id, destination id, and whether the destination path has content without state. Outputs are validated state values, JSON bytes, comparison outcomes, and human-readable reasons. +Inputs are destination state JSON, constructed catalog values, owner scopes, managed output paths, timestamps, and prune policy inputs. Outputs are validated catalog values, JSON bytes, managed path lists, owner-filtered output lists, missing-output repair projections, and prune candidate plans. ## Boundaries -The package does not inspect storage backends, mutate files, choose transfer policy, build publish outputs, generate URLs, or parse config. Publish planning consumes state comparison outcomes. +The package does not inspect storage backends, mutate files, choose workflow actions, build publish outputs, generate URLs, or parse config. Publish planning consumes parsed catalog state and helper projections. The external destination state contract is documented in `docs/integrations/destination-state.md`. ## Config Fields Used -`internal/state` uses config state mode and reconciliation mode constants for destination state validation and legacy state normalization. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers. +`internal/state` uses shared constants for catalog mode, output kinds, slug-like id validation, link validation, storage path validation, and source manifest validation. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers. ## Adapters Used @@ -26,42 +26,34 @@ None. ## State And Manifest Behavior -`.distributor.json` schema version is `2` for newly written single-owner state. Required fields are `pipeline_id`, `destination_id`, `published_at`, `created_at`, `updated_at`, `state.mode`, `reconciliation.mode`, `source.manifest`, and `outputs`. `distributor_version` and `links` are optional. +Current `.distributor.json` publish output uses schema version `4` catalog state. Required top-level fields are `schema_version`, `created_at`, `updated_at`, `state.mode`, and `outputs`; `distributor_version` is optional. -Schema version `1` state remains readable. Parsing infers `state.mode: single_owner`, `reconciliation.mode: replace`, top-level `created_at` and `updated_at` from `published_at`, and per-output timestamps from `published_at`. +Each catalog output record requires a clean path, pipeline id, destination id, source identity, `source` or `generated` kind, lowercase SHA-256 digest, non-negative size, and created/updated timestamps. Generated outputs require `source_path` and `transform`; copied source outputs must omit both. Stored URLs are optional and must pass `internal/link` validation. -Schema version `3` is shared-root state. It records `state.mode: shared_root`, shared state timestamps, owner records keyed by pipeline id and destination id, each owner's latest source manifest and reconciliation metadata, optional owner primary links, and output records for every managed path. Shared-root output records include owner ids and compact source identity fields for source id, digest, and creation time. +Embedded source identity records contain source manifest id, digest, and creation timestamp. Full source manifests are not embedded in catalog state. -Shared-root publish conversion is explicit. Compatible single-owner state for the same pipeline and destination can be projected into the current owner scope by publish execution. Unrelated single-owner state remains a conflict. +The package identifies schema versions older than the current catalog schema as superseded legacy state for publish planning. It rejects invalid JSON, malformed catalog state, and unsupported future schema versions. -Embedded source manifests are parsed and validated through `internal/bundle`, which delegates source manifest semantics to `pkg/bundle`. Output records require clean paths, `source` or `generated` kind, valid source paths, lowercase SHA-256 digests, non-negative sizes, created and updated timestamps, and transform ids for generated outputs. Stored URLs must pass `internal/link` validation. +The package provides helpers for finding catalog outputs by path, filtering outputs by owner, listing managed output paths, removing missing output records for one owner or every owner, and building owner-scoped prune candidates. -The package also provides helpers for finding output records by path, projecting planned publish outputs into timestamped state outputs, merging retained and newly planned output records, computing managed output paths from single-owner state, removing missing managed output records from single-owner state, and building prune candidates from managed outputs. - -For shared-root state, helpers parse either state shape, identify the current owner scope, return an owner's latest source manifest, list managed paths for one owner or all owners, detect path ownership conflicts, project planned owner outputs, merge one owner's planned outputs while preserving unrelated owners, replace one owner's outputs by removing that owner's omitted outputs, remove missing managed output records for either the current owner or every owner, and build owner-scoped prune candidates. - -Shared-root helper projections preserve output `created_at` for existing managed paths and use the current publication time for rewritten `updated_at`. Root-level `created_at` preservation is owned by publish execution. +Catalog helper projections preserve output `created_at` for existing managed paths and use the current publication time for rewritten `updated_at`. Root-level `created_at` preservation is owned by publish execution. ## Skip And Resume Behavior -Comparison is pure. It returns outcomes for absent state, unmanaged content, invalid state, pipeline/destination mismatch, same source manifest, older destination, newer destination, same-created digest conflict, and different source id conflict. Shared-root helpers expose structured output ownership conflicts. State code does not decide whether to skip, replace, take over, force, or fail; publish planning maps outcomes and conflicts to actions. +Catalog parsing and helper transformations are pure. State code does not decide whether to skip, upsert, replace, force, or fail; publish planning maps parsed state and storage observations to actions. -Shared-root comparison is owner-scoped. It compares only the owner keyed by the current pipeline id and destination id, treats an absent owner as absent destination state for that owner, and can compare compatible single-owner state for the current owner without converting unrelated single-owner state. +Missing-output removal helpers remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers. -Missing-output removal helpers are pure state transformations used by app-level state repair. They remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers. - -Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution and the `prune` command use the missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds. +Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution uses missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds. ## Failure Behavior -Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid state mode, invalid reconciliation mode, invalid embedded manifests, duplicate owners, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transforms, invalid URLs, invalid digests, negative sizes, and shared-root outputs whose owner is not registered. +Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid catalog mode, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transform metadata, invalid URLs, invalid digests, and negative sizes. ## Tests To Inspect -- `internal/state/distributor_test.go` -- `internal/state/shared_root_test.go` +- `internal/state/catalog_test.go` - `internal/state/prune_test.go` -- `internal/state/compare_test.go` - `internal/app/reconcile_state_test.go` - `internal/cli/reconcile_state_test.go` - `internal/publish/*_test.go` @@ -69,15 +61,13 @@ Parsing rejects invalid JSON, trailing data, missing required fields, invalid ti ## Architectural Invariants - `.distributor.json` is the destination sentinel and state record. -- Comparison does not mutate storage. -- Embedded source manifests use the source bundle contract. -- Newly written single-owner state uses schema version `2`. -- Schema version `1` state remains readable as replacement-mode single-owner state. -- Schema version `3` shared-root state is parsed and validated without converting unrelated single-owner state. -- Shared-root owner updates preserve unrelated owners. Publish planning removes taken-over output records before owner replacement when takeover policy allows a managed path to move owners. +- State helpers do not inspect or mutate storage. +- Source identity uses the source bundle contract. +- Newly written publish state uses schema version `4`. +- Superseded legacy schema handling is limited to identifying older state for publish planning. - Missing-output repair helpers preserve unrelated owner records and outputs. -- Prune planning uses output `updated_at` and preserves unrelated shared-root owners. -- Generated outputs always record a transform id. +- Prune planning uses output `updated_at` and preserves unrelated owners. +- Generated outputs always record a transform id and source path. - Output records always carry created and updated timestamps after parsing. - Stored URLs are optional and must be absolute HTTP or HTTPS URLs when present. - `distributor_version` is diagnostic metadata, not a comparison key. diff --git a/docs/operations.md b/docs/operations.md index 9cc8aa9..7a49a61 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -56,45 +56,33 @@ Destination path mapping controls where each source bundle is published beneath Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple bundles have the same timestamp, the source-root-relative bundle path in ascending order wins. -Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination state record. It records the pipeline id, destination id, publication time, state mode, reconciliation mode, source manifest, copied outputs, generated outputs, output timestamps, and optional public URL metadata. +Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination catalog. It records catalog schema version `4`, output owners, source identity for each output, output digests and sizes, timestamps, and optional URL metadata. `manifest.json` from the source bundle is not copied as destination state. -## Destination State And Retry Behavior +## Catalog Publish Behavior -`distributor` compares the source manifest to destination `.distributor.json` before writing: +`distributor` plans from the current source bundle, destination workflow, destination storage content, and `.distributor.json`. -- No destination state and no destination content: publish new outputs. -- Matching destination state: skip as already published. -- Older destination state for the same source id: replace if transfer policy allows it. -- Newer destination state for the same source id: skip by default, or replace when `transfer.on_destination_newer: replace` is configured. -- Valid managed state with an identity, source, same-created digest, or shared-root output-owner mismatch: replace when destination `takeover.mode` allows it or, for remaining valid managed conflicts, when `transfer.on_conflict: replace` is configured. -- Invalid destination state, identity, source, same-created digest, or shared-root output-owner mismatches not allowed by `takeover.mode` or `transfer.on_conflict: replace`: fail by default. -- Content without `.distributor.json`: fail as unmanaged content by default. +- No valid state and no destination content: publish new outputs. +- No valid state and existing destination content: fail as unmanaged unless `--force` is used. +- Valid catalog state with `workflow: additive`: write planned outputs and retain unrelated catalog outputs. +- Valid catalog state with `workflow: replacement`: write planned outputs and remove omitted outputs owned by the selected pipeline and destination. +- Planned output path exists in storage but is not recorded in valid catalog state - fail as unmanaged unless `--force` is used. +- Invalid destination state or unsupported future state - fail as conflict unless `--force` is used. +- Superseded legacy state - publish through the catalog planner and write schema version `4` state on success. -When destination state is older or newer than the source, `transfer.on_destination_older` and `transfer.on_destination_newer` control whether publication may proceed and `reconciliation.mode` controls how managed outputs are updated. +`workflow: additive` is the default. It is useful for archive roots, fan-out roots that intentionally receive disjoint outputs, and roots that accumulate managed outputs over time. -When `transfer.on_conflict: replace` permits `replace_conflict`, the destination is valid managed state but does not match the current source, pipeline, destination, or same-created digest. `replace_conflict` rewrites the affected managed destination state and does not retain omitted outputs through merge reconciliation. - -For takeover replacement, `reconciliation.mode: merge` does not retain omitted outputs from the previous source identity. The destination is rewritten as a managed replacement for the current source or shared-root owner. - -`reconciliation.mode: replace` is the default. It deletes only managed output paths recorded in `.distributor.json` plus the state file, verifies the destination bundle path is empty, then writes the newly planned outputs and state. The new state `outputs` array is exactly the newly planned output set. - -`reconciliation.mode: merge` retains prior managed outputs that are omitted from the new plan. It overwrites planned paths only when those paths are already recorded in existing state as managed. If a newly planned path already exists in storage but is not recorded in state, publication fails as an unmanaged path collision. The new state `outputs` array is the cumulative managed output set. - -For both modes, retained or overwritten paths are identified only from `.distributor.json`; unmanaged files are not adopted. - -For `state.mode: shared_root`, one destination root may contain outputs from multiple pipeline/destination owners. Comparisons, replacement, and merge retention are scoped to the current owner. Outputs owned by other owners are preserved unless a planned output path is owned by another valid owner and `takeover.mode` allows moving that path to the current owner. A planned path that exists in storage but is not recorded in state fails as unmanaged content by default. - -If `state.mode: shared_root` is configured on a destination whose existing single-owner state belongs to the same pipeline and destination, the next successful publish converts that state file to shared-root schema. Existing single-owner state for a different pipeline or destination remains a conflict. - -If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. In `replace` mode and takeover replacement, cleanup removes outputs written during that failed attempt. In same-source `merge` mode, cleanup removes only newly created outputs from that failed attempt; overwritten managed outputs are left in place because they previously belonged to the managed set. Operators should still inspect the destination after a failed write before retrying. +`workflow: replacement` is useful for stable latest-style roots where the current pipeline and destination should leave only the currently planned output set for that owner. Replacement workflow is normal managed behavior and does not require `--force`. Fan-out destinations are independent. If one destination fails after planning or execution begins, later destinations are still attempted. The command exits non-zero if any destination failed. +If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. Operators should still inspect the destination bundle path after a failed write before retrying. + ## Destination State Repair -Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the state record only; it does not restore missing files. +Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the catalog record only; it does not restore missing files. Preview the repair first: @@ -115,11 +103,9 @@ go run ./cmd/distributor reconcile-state \ --destination ``` -The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks each managed output path with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting. +The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks matching catalog output paths with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting. -Without `--dry-run`, it removes missing managed output records from valid state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid or mismatched state. - -For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, repair is scoped to the selected owner by default. Add `--all-owners` only when every owner in the selected shared-root state should have missing managed output records removed. +Without `--dry-run`, it removes missing managed output records from valid catalog state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid state. Add `--all-owners` only when every catalog owner inside the selected root should be repaired. ## Managed Output Pruning @@ -147,31 +133,26 @@ go run ./cmd/distributor prune \ The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, and plans from the selected destination's `retention.prune` policy. It uses managed output `updated_at` timestamps. When both `keep_latest` and `older_than` are configured, it preserves the newest `keep_latest` outputs before applying the age policy. -`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate. - -For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners. +`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths for the selected pipeline/destination owner, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate. ## Dry Runs And Output Review -`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, or SSH `known_hosts` entries. For reconciliation, dry runs report the same high-level action labels as execution; inspect the configured destination's `reconciliation.mode` to determine whether `replace_older` will replace the managed set or merge into it. - -For shared-root destinations, dry runs are owner-scoped. A `replace_older` action replaces or merges only the current owner according to `reconciliation.mode`; unrelated owners remain managed by the shared-root state. Paths owned by another owner fail as conflicts unless `takeover.mode` allows ownership transfer. +`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, notifier events, or SSH `known_hosts` entries. Review these action labels before publishing: -- `publish_new`: destination state is absent, or a shared-root owner is absent and planned paths are publishable. -- `replace_older`: destination state is older than the source. -- `replace_newer`: destination state is newer than the source and `transfer.on_destination_newer: replace` allows managed replacement. -- `replace_conflict`: destination state is valid managed state with a conflict and `transfer.on_conflict: replace` allows managed replacement. -- `replace_takeover`: destination state is valid managed state and `takeover.mode` allows replacement across an identity, source, or shared-root output-owner mismatch. -- `skip_same`: destination state already matches the source. -- `skip_destination_newer`: destination state is newer than the source and is skipped. +- `publish_new`: destination state is absent and the destination bundle path is empty. +- `upsert_additive`: additive workflow will write planned outputs into valid catalog state. +- `replace_catalog`: replacement workflow will write planned outputs and remove omitted outputs for the current owner. +- `skip_same`: no-op action value in the run output vocabulary. - `force_replace`: destructive catalog replacement selected because `--force` is present for unmanaged content, a planned unmanaged path collision, invalid state, or unsupported future state. -- `error`: planning or execution failed for that destination. +- `fail_unmanaged`: unmanaged destination content prevents publication. +- `fail_conflict`: invalid or unsupported destination state prevents publication. +- `error`: setup, planning, or execution failed for that destination. -Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced. +Text and JSON summaries count `publish_new`, `upsert_additive`, `replace_catalog`, `skip_same`, `force_replace`, `fail_unmanaged`, `fail_conflict`, and failed destinations separately. JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document. -Text and JSON summaries count `force_replace` separately from normal publish and catalog workflow actions. JSON output includes warnings, pipeline summaries, destination action records, output records, URLs when configured, final counters, and partial failure details. Fatal setup failures such as unreadable config or invalid secrets do not produce a JSON result document. +Fixed destinations add fixed-path warnings during dry runs, including the selected source bundle and replacement warnings when the destination root would be replaced. For fixed destinations, the resolved destination bundle path is the backend root. ## Forced Replacement Workflow @@ -186,8 +167,6 @@ Forced replacement can claim a non-empty destination path with no valid `.distri Forced replacement deletes the current destination bundle path before writing planned outputs and schema version `4` catalog state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the configured backend root, so a forced replacement can clear that configured root. Dry-run text and JSON output report that root as `target=.` or `destination_path: "."`. -Preview with `--dry-run --force` and confirm the destination path before applying. - `--force` applies only to the current invocation. There is no config field that enables forced replacement by default. ## HTTP Upload Operation @@ -226,9 +205,9 @@ curl -X POST http://127.0.0.1:8080/v1/pipelines/example-http-upload/upload \ --data-binary @bundle.tar.gz ``` -Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md) for the copyable producer implementation guide. +Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md). -The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`. Set `DISTRIBUTOR_EXAMPLE_UPLOAD_PIPELINE_ID` or pass a second argument to use another configured upload pipeline. It generates an idempotency key by default; set `DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY` when retrying the same producer run across separate process runs. +The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`. ```sh go run ./examples/upload-client @@ -250,9 +229,9 @@ Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Co Upload admission is bounded by `server.http.queue_size`. Publication concurrency is bounded by `server.http.max_concurrency`, and the coordinator does not run two uploads for the same pipeline at the same time. -`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`. If another request with the same key is still being staged before its manifest is known, the server returns a retryable `409 Conflict`. Idempotency records are memory-only and expire with completed upload status records. +`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`. -The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size. The implementation also caps extracted file count. +The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size. The default bind address is private loopback. Put TLS, public routing, rate limiting, and external access policy in a reverse proxy or deployment layer. @@ -287,11 +266,10 @@ Use these recovery boundaries: - For source validation failures, regenerate the source bundle and manifest together. - For an empty or missing destination, rerun after fixing config or storage access. - For unmanaged destination content, move unrelated files aside or use a different destination path before publishing. -- For shared-root ownership conflicts, change one owner so it writes a different destination path, use a separate destination root, or configure `takeover.mode` when the current owner should take over valid managed output paths. +- For invalid or unsupported destination state, inspect `.distributor.json`; use `--force` only after dry-run review confirms bounded replacement is intended. - For missing managed output files recorded in state, run `reconcile-state --dry-run`, then apply `reconcile-state` if the missing files should no longer be considered managed. - For configured retention cleanup, run `prune --dry-run`, then apply `prune --apply` after reviewing the managed output list. -- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`. In merge mode, retained outputs may be intentional managed outputs from the prior state. -- For state conflicts, verify the source, pipeline, destination, and existing `.distributor.json` before considering `--force`. +- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`. - For HTTP upload failures, inspect `/runs/` while retained; after expiry or restart, rely on destination state and logs/output from the publishing run. Do not edit `.distributor.json` during normal recovery. Treat it as the managed state record used for comparison and safe cleanup. diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index bd5c302..0a5f755 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -96,18 +96,15 @@ The source manifest should remain minimal. Routing, destination selection, publi Each destination bundle path is managed by `.distributor.json`. This file is both the destination sentinel and the destination state record. -Publish execution writes single-owner or shared-root destination state according to destination `state.mode`. In shared-root state, one `.distributor.json` records multiple pipeline/destination owners and every managed output carries its owner identity. +Publish execution writes catalog destination state. One `.distributor.json` records all managed outputs under the destination bundle path, and each output carries its owning pipeline id and destination id. -Single-owner state records: +Catalog state records: - `distributor` state schema version; -- pipeline id; -- destination id; -- publication timestamp; - state creation and update timestamps; -- single-owner state mode; -- reconciliation mode; -- the normalized source manifest used for publication; +- catalog state mode; +- owner identity for each managed output; +- compact source identity for each managed output; - metadata for copied source outputs; - metadata for generated outputs, such as HTML files; - optional URL metadata for published outputs; @@ -117,40 +114,23 @@ A representative destination state file is: ```json { - "schema_version": 2, + "schema_version": 4, "distributor_version": "0.1.0", - "pipeline_id": "weather-daily", - "destination_id": "static-html", - "published_at": "2026-05-30T11:12:00Z", "created_at": "2026-05-30T11:12:00Z", "updated_at": "2026-05-30T11:12:00Z", "state": { - "mode": "single_owner" - }, - "reconciliation": { - "mode": "replace" - }, - "source": { - "manifest": { - "schema_version": 1, - "id": "weather.daily.brentwood.2026-05-30", - "digest": "sha256:...", - "created": "2026-05-30T11:10:00Z", - "files": [ - { - "path": "report.md", - "sha256": "sha256:...", - "size": 12345 - } - ] - } - }, - "links": { - "primary_url": "https://reports.example.com/weather-daily/" + "mode": "catalog" }, "outputs": [ { "path": "index.html", + "pipeline_id": "weather-daily", + "destination_id": "static-html", + "source": { + "id": "weather.daily.brentwood.2026-05-30", + "digest": "sha256:...", + "created": "2026-05-30T11:10:00Z" + }, "kind": "generated", "source_path": "report.md", "transform": "markdown_to_html", @@ -167,13 +147,11 @@ A representative destination state file is: Destination comparison rules are based on `.distributor.json`: - No `.distributor.json`: publish normally only if the destination bundle path is empty. -- Existing state embeds the same normalized source manifest: skip as already published. -- Existing state has the same source id and an older source `created`: replace, subject to destructive-operation safety rules. -- Existing state has the same source id and a newer source `created`: skip by default, or replace when `transfer.on_destination_newer: replace` is configured. -- Existing state has the same source id and same `created` but different digest: fail by default, or replace as valid managed conflict state when `transfer.on_conflict: replace` is configured. -- Existing valid managed state has a different source id, pipeline id, destination id, or shared-root output owner: replace when destination `takeover.mode` permits that ownership transfer, or replace as valid managed conflict state when `transfer.on_conflict: replace` is configured; otherwise fail as a conflict. - -For older destination state, destination `reconciliation.mode` controls output updates. `replace` rewrites the managed output set to match the new plan. `merge` retains omitted managed outputs, overwrites only existing managed paths, and rejects planned paths that collide with unmanaged storage content. +- Existing catalog state with additive workflow: write planned outputs and retain unrelated managed outputs. +- Existing catalog state with replacement workflow: write planned outputs and remove omitted outputs for the current pipeline and destination owner. +- Planned paths that collide with unmanaged storage content fail by default. +- Invalid or unsupported destination state fails by default. +- Explicit forced replacement may clear the bounded destination bundle path after dry-run review. ## Publication and Transform Policy @@ -264,8 +242,7 @@ Pipeline configuration should express: - per-destination transform policy; - per-destination public link policy; - validation behavior; -- per-destination takeover behavior; -- destination conflict/replacement behavior. +- per-destination workflow and retention behavior. ## Modules and Registries diff --git a/docs/policy/development.md b/docs/policy/development.md index c3229ec..10549f1 100644 --- a/docs/policy/development.md +++ b/docs/policy/development.md @@ -18,7 +18,7 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`. - `internal/adapters/ssh`: SSH/SFTP backend. - `internal/adapters/s3`: S3-compatible object storage backend. - `internal/storage/fake`: in-memory backend for tests. -- `internal/publish`: destination inspection, output planning, takeover planning, reconciliation, execution, managed cleanup, and explicit forced replacement. +- `internal/publish`: destination inspection, output planning, catalog workflow planning, execution, managed cleanup, and explicit forced replacement. - `internal/transform`: transform interface and registry. - `internal/transform/markdown`: Markdown-to-HTML transform. - `internal/notify`: notification interface and current no-op notifier. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 5260a5d..ed30998 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -24,7 +24,7 @@ Reference: [Configuration](config.md#config-file-loading). Symptom: `parse config ... field not found`. -Likely cause: the YAML contains a key that is not part of the implemented config schema. +Likely cause: the YAML contains a key that is not part of the implemented config schema. Legacy destination policy keys `state`, `reconciliation`, `takeover`, and `transfer` are rejected. Diagnostic: @@ -32,9 +32,9 @@ Diagnostic: go run ./cmd/distributor run --config --dry-run ``` -Safe fix: remove or rename unsupported fields using the canonical config reference. +Safe fix: remove unsupported fields using the canonical config reference. Destination behavior is configured with `workflow`, `publish`, `transform`, `path_mapping`, `links`, and `retention`. -Reference: [Configuration](config.md). +Reference: [Configuration](config.md#destination-fields). ## Backend Name Or Placement Is Invalid @@ -71,9 +71,9 @@ Reference: [CLI](cli.md#validate). ## Reconcile-State Selector Is Missing Or Wrong -Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "" not found`, `destination not found`, or `state owner is ... not ...`. +Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "" not found`, or `destination not found`. -Likely cause: the command did not identify one configured destination root, or the selected root contains state for a different single-owner pipeline/destination. +Likely cause: the command did not identify one configured destination root. Diagnostic: @@ -83,15 +83,15 @@ rg -n 'pipelines:|destinations:|id:' cat /.distributor.json ``` -Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. For unrelated single-owner state, use the correct config selector or a separate destination root; `reconcile-state` will not take over mismatched state. +Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Reference: [CLI](cli.md#reconcile-state). ## Prune Selector Or Mode Is Missing Or Wrong -Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "" not found`, `destination not found`, or `state owner is ... not ...`. +Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "" not found`, or `destination not found`. -Likely cause: the command did not identify one configured destination root, did not choose exactly one execution mode, or the selected root contains state for a different single-owner pipeline/destination. +Likely cause: the command did not identify one configured destination root or did not choose exactly one execution mode. Diagnostic: @@ -101,7 +101,7 @@ rg -n 'retention:|prune:|pipelines:|destinations:|id:' cat /.distributor.json ``` -Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both. For unrelated single-owner state, use the correct config selector or a separate destination root; `prune` will not take over mismatched state. +Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both. Reference: [CLI](cli.md#prune). @@ -201,22 +201,39 @@ Safe fix: choose an empty destination path, move unrelated files aside, or previ Reference: [Operations](operations.md#forced-replacement-workflow). -## Destination State Conflicts With Source +## Destination State Is Invalid Or Unsupported -Symptom: `fail_conflict`, `destination source id differs`, `same id and created time but different digest`, `pipeline id ... does not match`, or `destination id ... does not match`. +Symptom: `fail_conflict`, `parse distributor state`, `state schema_version must be 4`, or `unsupported future destination state`. -Likely cause: `.distributor.json` belongs to a different pipeline, destination, source id, shared-root output owner, or same-created source with different content. Valid managed identity, source, and shared-root output-owner mismatches can publish as `replace_takeover` when destination `takeover.mode` allows them, or as `replace_conflict` when `transfer.on_conflict: replace` allows managed conflict replacement. +Likely cause: `.distributor.json` is invalid JSON, has invalid catalog fields, or uses an unsupported future schema. Diagnostic: ```sh cat /.distributor.json -go run ./cmd/distributor inspect +go run ./cmd/distributor run --config --dry-run --format json ``` -Safe fix: verify the source and destination are intended to match. Use a separate destination path for unrelated content. Use `--force` only for exceptional replacement of unmanaged content, invalid state, unsupported future state, or other cases that `run --dry-run --force` reports as `force_replace`. +Safe fix: restore a valid catalog state file from backup, choose a different destination path, or use `--force` only after `run --dry-run --force` reports the intended bounded `force_replace`. -Reference: [Operations](operations.md#destination-state-and-retry-behavior). +Reference: [Operations](operations.md#forced-replacement-workflow). + +## Destination Uses Superseded Legacy State + +Symptom: dry-run reports a normal catalog action against an older `.distributor.json`, or `reconcile-state` / `prune` reports that the destination state schema is superseded. + +Likely cause: the destination contains a state file written by an older implementation. Publish planning can replace it with catalog state on successful `run`, but maintenance commands only operate on current catalog state. + +Diagnostic: + +```sh +cat /.distributor.json +go run ./cmd/distributor run --config --dry-run +``` + +Safe fix: preview the publish plan, then run publication if the destination path is correct. The successful run writes schema version `4` catalog state. + +Reference: [Destination State Contract](integrations/destination-state.md). ## Destination State References Missing Managed Outputs @@ -242,7 +259,7 @@ Reference: [Operations](operations.md#destination-state-repair). Symptom: `prune --dry-run` reports `planned=0` or JSON `planned_outputs: []`. -Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected shared-root owner has no eligible outputs. +Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected pipeline/destination owner has no eligible outputs. Diagnostic: @@ -260,22 +277,6 @@ Safe fix: verify the selected destination's retention policy and the managed out Reference: [Operations](operations.md#managed-output-pruning). -## Destination Is Newer Than Source - -Symptom: `skip_destination_newer`. - -Likely cause: the destination state records a source manifest with a later `created` timestamp than the current source. - -Diagnostic: - -```sh -go run ./cmd/distributor run --config --dry-run --format json -``` - -Safe fix: keep the default skip behavior unless replacement is intentional. To replace newer valid managed state, configure `transfer.on_destination_newer: replace`, preview with `--dry-run`, then publish without `--force`. - -Reference: [Operations](operations.md#forced-replacement-workflow). - ## Forced Replacement Appears In A Plan Symptom: dry-run output includes `force_replace`. @@ -294,9 +295,9 @@ Reference: [Operations](operations.md#forced-replacement-workflow). ## Output Path Collision -Symptom: `destination output path collision` or `merge output path ... exists but is not managed by destination state`. +Symptom: `destination output path collision` or `destination output path ... exists but is not managed by catalog state`. -Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`. For merge reconciliation, it can also mean a planned output path already exists in storage but is not recorded in `.distributor.json` as managed. +Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`, or a planned output path already exists in storage but is not catalog-managed. Diagnostic: @@ -304,44 +305,10 @@ Diagnostic: go run ./cmd/distributor run --config --dry-run ``` -Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For merge reconciliation, move unmanaged content aside, choose another destination path, or use replacement/forced replacement only when deleting the destination bundle path is intended. +Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For unmanaged storage collisions, move the unmanaged file aside, choose another destination path, or use forced replacement only when deleting the destination bundle path is intended. Reference: [Configuration](config.md#publish-and-transform-policy). -## Shared-Root Ownership Conflict - -Symptom: `fail_conflict` with a reason like `destination output path ... is owned by /`. - -Likely cause: a `state.mode: shared_root` destination planned an output path already recorded in `.distributor.json` for another pipeline/destination owner, and `takeover.mode` does not allow that managed path to move to the current owner. - -Diagnostic: - -```sh -cat /.distributor.json -go run ./cmd/distributor run --config --dry-run -``` - -Safe fix: change one owner so it writes a different output path, use a separate destination root, or intentionally replace the whole destination root only after previewing with `--dry-run --force`. - -Reference: [Operations](operations.md#destination-state-and-retry-behavior). - -## Shared-Root Unmanaged Path Collision - -Symptom: `fail_unmanaged` with a reason like `destination output path ... exists but is not managed by catalog state`. - -Likely cause: a `state.mode: shared_root` destination planned a new output path that already exists in storage but is not recorded as managed in `.distributor.json`. - -Diagnostic: - -```sh -find -maxdepth 2 -print -cat /.distributor.json -``` - -Safe fix: move the unmanaged file aside, change the planned output path, or use forced replacement only when deleting the configured destination root is intended. - -Reference: [Operations](operations.md#forced-replacement-workflow). - ## Run Failed After Writing Some Files Symptom: a destination write failed and the command exited non-zero after partial work. @@ -354,9 +321,9 @@ Diagnostic: find -maxdepth 2 -print ``` -Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. In same-source merge mode, previously managed retained or overwritten outputs may remain intentionally. Rerun `--dry-run` before publishing again. +Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. Rerun `--dry-run` before publishing again. -Reference: [Operations](operations.md#destination-state-and-retry-behavior). +Reference: [Operations](operations.md#catalog-publish-behavior). ## JSON Run Result Has `ok: false` @@ -378,240 +345,14 @@ Reference: [CLI](cli.md#output-and-exit-behavior). Symptom: `load secrets directory ... no such file or directory`, `permission denied`, or `secret filename ... is invalid`. -Likely cause: `secrets.directory` points to a missing or unreadable directory, or it contains a regular file whose name is not a valid credential variable name. +Likely cause: `secrets.directory` points to a missing or unreadable directory, or contains a filename that cannot be used as a credential variable name. Diagnostic: ```sh -ls -ld -find -maxdepth 1 -type f -printf '%f\n' +ls -la ``` -Safe fix: mount or create the directory, adjust permissions for the service user, or rename/remove invalid secret files. Secret filenames must match `[A-Za-z_][A-Za-z0-9_]*`. +Safe fix: create the directory, fix permissions, or rename secret files to valid environment-variable-style names. Reference: [Configuration](config.md#secrets). - -## Credential Variable Is Missing Or Empty - -Symptom: `credential environment variable ... is not set`, `credential environment variable ... is empty`, or S3 authentication errors such as `AccessDenied`, `InvalidAccessKeyId`, or `SignatureDoesNotMatch`. - -Likely cause: configured S3 credential variable names are not available through the process environment or `secrets.directory`, are empty, or do not authorize the requested bucket/prefix. - -Diagnostic: - -```sh -env | cut -d= -f1 | rg '^(|)$' -ls -l -``` - -Safe fix: provide both configured S3 credential values, correct IAM/service permissions, or omit explicit credential fields to use the AWS SDK default credential chain. - -Reference: [Configuration](config.md#s3-compatible-backend). - -## Secret File Is Ignored In Favor Of Environment - -Symptom: `secret ... ignored because the real environment already has that variable`. - -Likely cause: the same credential name exists in the process environment and `secrets.directory` with different values. - -Diagnostic: - -```sh -env | cut -d= -f1 | rg '^$' -ls -l / -``` - -Safe fix: remove one source of the credential or intentionally keep the process environment value. `distributor` does not print either value. - -Reference: [Operations](operations.md#secrets-operation). - -## SSH Auth Is Not Configured - -Symptom: `no SSH auth methods configured`. - -Likely cause: no SSH agent is available and `ssh_key_file` is missing or unreadable. - -Diagnostic: - -```sh -test -n "$SSH_AUTH_SOCK" && ssh-add -l -ls -l -``` - -Safe fix: start an SSH agent with an appropriate key loaded, or configure a readable private key with `ssh_key_file`. - -Reference: [Configuration](config.md#sshsftp-backend). - -## SSH Host Key Fails - -Symptom: `host key ... is unknown`, `known_hosts is required`, or `host key ... has changed`. - -Likely cause: strict host key checking has no trusted key, `accept-new` cannot persist a new key, or the remote host key differs from the stored key. - -Diagnostic: - -```sh -ls -l -ssh-keygen -F -f -``` - -Safe fix: pre-populate `known_hosts` for `strict`, configure a writable `known_hosts` path for `accept-new`, or verify the server identity before updating a changed key. Do not disable host key checking to bypass an unexpected changed key. - -Reference: [Operations](operations.md#sshsftp). - -## S3 Prefix Is Invalid - -Symptom: `prefix must be a clean relative slash-separated path`. - -Likely cause: the S3 prefix contains traversal, dot segments, empty segments, or backslashes after leading and trailing slashes are trimmed. - -Diagnostic: - -```sh -go run ./cmd/distributor run --config --dry-run -``` - -Safe fix: use a clean relative prefix such as `reports/archive`, or omit `prefix`. - -Reference: [Configuration](config.md#s3-compatible-backend). - -## S3 Location Or Connectivity Fails - -Symptom: `NoSuchBucket`, `InvalidBucketName`, `not_found`, endpoint connection failures, or TLS/network errors. - -Likely cause: endpoint, bucket, prefix, region, path-style mode, network routing, or credentials are wrong for the service. - -Diagnostic: - -```sh -go run ./cmd/distributor run --config --dry-run -curl -I -``` - -Safe fix: verify `endpoint`, `bucket`, `region`, `prefix`, and `force_path_style`. For S3-compatible services, keep `force_path_style: true` unless the service requires virtual-host addressing. Distributor does not provide an insecure TLS bypass setting. - -Reference: [Operations](operations.md#s3-compatible-storage). - -## HTTP Server Cannot Bind - -Symptom: `bind HTTP server ... address already in use`. - -Likely cause: another process is listening on `server.http.bind`. - -Diagnostic: - -```sh -ss -ltnp | rg '' -``` - -Safe fix: stop the conflicting process or configure a different bind address. - -Reference: [Configuration](config.md#serverhttp). - -## HTTP Upload Token Is Missing Or Duplicated - -Symptom: `upload token environment variable ... is not set`, `... is empty`, or `upload token environment variables ... resolve to the same value`. - -Likely cause: a top-level upload token record references a missing or empty `token_env`, or two token records resolve to the same bearer token. - -Diagnostic: - -```sh -rg -n 'token_env:' -env | cut -d= -f1 | rg '^$' -ls -l / -``` - -Safe fix: provide one distinct non-empty token value per upload token record through the process environment or `secrets.directory`. Do not put literal tokens in YAML. - -Reference: [Configuration](config.md#upload_tokens). - -## Upload Request Is Unauthorized - -Symptom: `POST /v1/pipelines//upload` returns `401`. - -Likely cause: the request lacks `Authorization: Bearer `, has an empty token, or uses a token that does not match any configured upload token record. - -Diagnostic: - -```sh -curl -i -X POST http://127.0.0.1:8080/v1/pipelines//upload \ - -H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \ - -H "Content-Type: application/x-tar" \ - --data-binary @bundle.tar -``` - -Safe fix: use the token value resolved by the configured `token_env`. Do not include token values in logs or tickets. - -Reference: [Operations](operations.md#http-upload-operation). - -## Upload Request Is Forbidden - -Symptom: `POST /v1/pipelines//upload` returns `403`. - -Likely cause: the bearer token is valid, but its configured `allow_pipelines` list does not include the requested upload pipeline. - -Diagnostic: - -```sh -rg -n 'upload_tokens:|allow_pipelines:|id:' -``` - -Safe fix: request the intended pipeline id, or update the token allowlist to include the configured `http_upload` pipeline that this producer may submit to. - -Reference: [Configuration](config.md#upload_tokens). - -## Upload Request Is Rejected Before A Run ID - -Symptom: `POST /v1/pipelines//upload` returns `400`, `413`, `415`, or `503`. - -Likely cause: the request path has an invalid pipeline id, included a `pipeline` or `pipeline_id` query, archive content is malformed, the body exceeds size limits, content type is unsupported, or the in-memory upload queue is full. - -Diagnostic: - -```sh -tar -tf bundle.tar -tar -tzf bundle.tar.gz -rg -n 'max_upload_size|queue_size|max_concurrency' -``` - -Safe fix: send one valid tar or tar.gz source bundle archive to `/v1/pipelines//upload` with `Content-Type: application/x-tar`, `application/gzip`, or `application/x-gzip`; remove pipeline query parameters; reduce archive size or raise the configured limit; retry after queue pressure drops. - -Reference: [Operations](operations.md#http-upload-operation). - -## Upload Idempotency Conflict - -Symptom: `POST /v1/pipelines//upload` returns `409`. - -Likely cause: the request reused an `Idempotency-Key` for the same token id and pipeline id with a different source manifest, or another request with the same key is still being staged before its manifest is known. - -Diagnostic: - -```sh -curl -i -X POST http://127.0.0.1:8080/v1/pipelines//upload \ - -H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \ - -H "Content-Type: application/gzip" \ - -H "Idempotency-Key: " \ - --data-binary @bundle.tar.gz -``` - -Safe fix: if the response includes `"retryable":true`, retry the same upload later with the same key. Otherwise, inspect the producer operation and use the same key only for the same source bundle. - -Reference: [HTTP Upload API Contract](integrations/http-upload.md). - -## Upload Status Is Missing - -Symptom: `GET /runs/` returns `404`. - -Likely cause: the run id is wrong, the process restarted, or the retained status record expired after `server.http.retention`. - -Diagnostic: - -```sh -curl -i http://127.0.0.1:8080/runs/ -rg -n 'retention:' -``` - -Safe fix: use the exact `run_id` returned by upload admission. Increase retention if operators need a longer status window. - -Reference: [Operations](operations.md#http-upload-operation). diff --git a/examples/shared-root.yml b/examples/additive-workflow.yml similarity index 59% rename from examples/shared-root.yml rename to examples/additive-workflow.yml index a7aa326..9073c16 100644 --- a/examples/shared-root.yml +++ b/examples/additive-workflow.yml @@ -1,27 +1,25 @@ pipelines: - - id: example-shared-root-source + - id: example-additive-source source: backend: local path: examples/source-bundle destinations: - - id: shared-local-root + - id: catalog-source backend: local - path: workspace/published/shared-root - state: - mode: shared_root + path: workspace/published/additive-workflow + workflow: additive publish: source: true html: false - - id: example-shared-root-html + - id: example-additive-html source: backend: local path: examples/source-bundle destinations: - - id: shared-local-root + - id: catalog-html backend: local - path: workspace/published/shared-root - state: - mode: shared_root + path: workspace/published/additive-workflow + workflow: additive publish: source: false html: true diff --git a/examples/archive-and-latest.yml b/examples/archive-and-latest.yml index 706d5d8..3675b30 100644 --- a/examples/archive-and-latest.yml +++ b/examples/archive-and-latest.yml @@ -7,6 +7,7 @@ pipelines: - id: local-source-archive backend: local path: workspace/published/archive-and-latest/archive + workflow: additive path_mapping: mode: preserve_relative publish: @@ -15,6 +16,7 @@ pipelines: - id: local-html-latest backend: local path: workspace/published/archive-and-latest/latest + workflow: replacement path_mapping: mode: fixed links: diff --git a/examples/merge-reconciliation.yml b/examples/replacement-workflow.yml similarity index 53% rename from examples/merge-reconciliation.yml rename to examples/replacement-workflow.yml index 35f0f68..58605a9 100644 --- a/examples/merge-reconciliation.yml +++ b/examples/replacement-workflow.yml @@ -1,18 +1,19 @@ pipelines: - - id: example-merge-reconciliation + - id: example-replacement-workflow source: backend: local path: examples/source-bundle destinations: - - id: local-merge-html + - id: local-latest-html backend: local - path: workspace/published/merge-reconciliation + path: workspace/published/replacement-workflow + workflow: replacement + path_mapping: + mode: fixed publish: source: false html: true transform: markdown_to_html: enabled: true - mode: sidecar - reconciliation: - mode: merge + mode: index diff --git a/internal/config/load_test.go b/internal/config/load_test.go index 69a485f..33abc88 100644 --- a/internal/config/load_test.go +++ b/internal/config/load_test.go @@ -958,17 +958,14 @@ pipelines: } func TestExampleConfigsLoad(t *testing.T) { - for _, path := range []string{ - "../../examples/local-to-local.yml", - "../../examples/local-publish.yml", - "../../examples/local-html.yml", - "../../examples/local-index.yml", - "../../examples/fan-out.yml", - "../../examples/archive-and-latest.yml", - "../../examples/http-upload-local.yml", - "../../examples/ssh-destination.yml", - "../../examples/s3-destination.yml", - } { + paths, err := filepath.Glob("../../examples/*.yml") + if err != nil { + t.Fatalf("glob examples: %v", err) + } + if len(paths) == 0 { + t.Fatal("no example configs found") + } + for _, path := range paths { t.Run(path, func(t *testing.T) { if _, err := LoadFile(path); err != nil { t.Fatalf("LoadFile(%q) error = %v", path, err)