Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e3b92a3b5d | |||
| c804fd604a | |||
| ea562c1c3a | |||
| 6daddad543 | |||
| c67ecf86a9 | |||
| 2abd09bde3 | |||
| de6723c5de | |||
| cb9502f790 | |||
| 9afb3550c4 | |||
| 89169f810f | |||
| eb86cf9ab6 | |||
| ef0b6c1056 | |||
| 93821ea6f9 | |||
| b7db3993fb | |||
| c04432e40b | |||
| 8f3ef33f18 | |||
| ee6a351960 | |||
| 29f01da37b | |||
| bd5892d1f2 | |||
| ce43a6044a | |||
| 1c5d7198e3 | |||
| 9d4694c6d8 | |||
| 033b2e5015 | |||
| 4fa7d1ebb5 | |||
| f98e528c90 | |||
| 25fbfc4677 |
@@ -10,11 +10,12 @@ Run the maintained local example:
|
||||
go run ./cmd/distributor run --config examples/local-publish.yml
|
||||
```
|
||||
|
||||
Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build, write, parse, and validate local source bundles with the same manifest contract used by the CLI. They can use `gitea.maximumdirect.net/eric/distributor/pkg/upload` to build or validate a bundle and submit it to `distributor serve` with bearer authentication and idempotency keys. See [Source bundle contract](docs/integrations/source-bundle.md) and [HTTP upload contract](docs/integrations/http-upload.md).
|
||||
Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/upload` and `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to submit compatible bundles to `distributor serve`. See [Upstream producer integration](docs/consumers/api.md).
|
||||
|
||||
- [CLI reference](docs/cli.md)
|
||||
- [Configuration reference](docs/config.md)
|
||||
- [Operations guide](docs/operations.md)
|
||||
- [Consumer API guide](docs/consumers/api.md)
|
||||
- [Troubleshooting](docs/troubleshooting.md)
|
||||
- [Integration contracts](docs/integrations/source-bundle.md)
|
||||
- [Development architecture](docs/policy/architecture.md)
|
||||
|
||||
85
docs/cli.md
85
docs/cli.md
@@ -21,6 +21,8 @@ distributor [--help]
|
||||
distributor help
|
||||
distributor version [--format text|json]
|
||||
distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
|
||||
distributor reconcile-state --config <path> --pipeline <id> --destination <id> [--all-owners] [--dry-run] [--format text|json]
|
||||
distributor prune --config <path> --pipeline <id> --destination <id> (--dry-run|--apply) [--format text|json]
|
||||
distributor serve [--config <path>]
|
||||
distributor validate [--format text|json] <path>
|
||||
distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
|
||||
@@ -33,6 +35,8 @@ distributor manifest create --id <bundle-id> [options] <bundle-path>
|
||||
|
||||
- `version` prints the application name and version.
|
||||
- `run` executes configured pipelines against their destinations.
|
||||
- `reconcile-state` repairs destination state records for missing managed outputs.
|
||||
- `prune` deletes managed outputs selected by the destination retention policy when `--apply` is supplied.
|
||||
- `serve` starts the authenticated HTTP upload API defined by the configuration file.
|
||||
- `validate` checks a local bundle path or a configured source bundle.
|
||||
- `inspect` reports manifest and file metadata for a local bundle path or a configured source bundle.
|
||||
@@ -46,7 +50,7 @@ distributor manifest create --id <bundle-id> [options] <bundle-path>
|
||||
|
||||
### Common Output Format
|
||||
|
||||
`--format text|json` is supported by `version`, `run`, `validate`, `inspect`, and `manifest create`.
|
||||
`--format text|json` is supported by `version`, `run`, `reconcile-state`, `prune`, `validate`, `inspect`, and `manifest create`.
|
||||
|
||||
- `text` is the default human-readable output.
|
||||
- `json` emits one JSON document for successful command execution.
|
||||
@@ -73,6 +77,36 @@ distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
|
||||
|
||||
`run` accepts no positional arguments.
|
||||
|
||||
### `reconcile-state`
|
||||
|
||||
```sh
|
||||
distributor reconcile-state --config <path> --pipeline <id> --destination <id> [--all-owners] [--dry-run] [--format text|json]
|
||||
```
|
||||
|
||||
- `--config <path>` loads the pipeline configuration and is required.
|
||||
- `--pipeline <id>` selects the pipeline used to identify the destination root and is required.
|
||||
- `--destination <id>` selects the destination root and is required.
|
||||
- `--all-owners` repairs missing managed output records for every owner in a shared-root state file. Without it, shared-root repair is scoped to the selected pipeline and destination owner.
|
||||
- `--dry-run` reports repairs without rewriting `.distributor.json`.
|
||||
- `--format text|json` selects human-readable or machine-readable output.
|
||||
|
||||
Without `--dry-run`, `reconcile-state` applies state repair by removing records for managed outputs that no longer exist in storage. It reports unmanaged entries but does not delete destination files, adopt unmanaged files, or validate output digests. The command accepts no positional arguments.
|
||||
|
||||
### `prune`
|
||||
|
||||
```sh
|
||||
distributor prune --config <path> --pipeline <id> --destination <id> (--dry-run|--apply) [--format text|json]
|
||||
```
|
||||
|
||||
- `--config <path>` loads the pipeline configuration and is required.
|
||||
- `--pipeline <id>` selects the pipeline used to identify the destination root and owner scope.
|
||||
- `--destination <id>` selects the destination root and owner scope.
|
||||
- `--dry-run` reports planned managed-output deletes without deleting outputs or rewriting `.distributor.json`.
|
||||
- `--apply` deletes planned managed outputs and rewrites `.distributor.json` after confirmed deletes.
|
||||
- `--format text|json` selects human-readable or machine-readable output.
|
||||
|
||||
Exactly one of `--dry-run` or `--apply` is required. The command uses only the selected destination's configured `retention.prune` policy; it does not accept one-off retention overrides. Apply mode deletes only planned managed output paths, preserves unmanaged files, and preserves `.distributor.json` even when no managed outputs remain.
|
||||
|
||||
### `serve`
|
||||
|
||||
```sh
|
||||
@@ -174,6 +208,53 @@ go run ./cmd/distributor run --config examples/local-publish.yml
|
||||
|
||||
Use `--format json` when automation needs structured run results. Use `--force` only when the operator has reviewed the destination state conflict and intentionally wants to continue.
|
||||
|
||||
### Repair Destination State Records
|
||||
|
||||
Preview missing managed output records for one configured destination:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor reconcile-state \
|
||||
--config examples/local-publish.yml \
|
||||
--pipeline example-source-bundle \
|
||||
--destination local-archive \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
Apply the repair after reviewing the report:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor reconcile-state \
|
||||
--config examples/local-publish.yml \
|
||||
--pipeline example-source-bundle \
|
||||
--destination local-archive
|
||||
```
|
||||
|
||||
Use `--all-owners` only for shared-root destination state when all owners inside the selected root should be repaired.
|
||||
|
||||
### Prune Managed Outputs
|
||||
|
||||
Preview managed outputs selected by the configured retention policy:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune \
|
||||
--config examples/local-publish.yml \
|
||||
--pipeline example-source-bundle \
|
||||
--destination local-archive \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
Apply after reviewing the report:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune \
|
||||
--config examples/local-publish.yml \
|
||||
--pipeline example-source-bundle \
|
||||
--destination local-archive \
|
||||
--apply
|
||||
```
|
||||
|
||||
Use `--format json` when automation needs structured prune results.
|
||||
|
||||
### Run HTML And Fan-Out Examples
|
||||
|
||||
```sh
|
||||
@@ -217,6 +298,8 @@ Text output is optimized for direct operator use. JSON output is optimized for a
|
||||
- Use `validate` before `run` when checking a bundle supplied by another process.
|
||||
- Use `inspect --format json` when automation needs manifest metadata, normalized file details, or checksum information.
|
||||
- Use `run --dry-run` before publishing to review destination actions.
|
||||
- Use `reconcile-state --dry-run` to inspect missing managed output records before repairing destination state.
|
||||
- Use `prune --dry-run` before `prune --apply` to review configured retention deletes.
|
||||
- Use [Configuration](config.md) for schema and default details.
|
||||
- Use [Troubleshooting](troubleshooting.md) for common errors and corrective action.
|
||||
- Use [Operations](operations.md) for HTTP upload operation, state files, and recovery workflows.
|
||||
|
||||
154
docs/config.md
154
docs/config.md
@@ -17,7 +17,7 @@ YAML decoding rejects unknown fields. Defaults are applied after decoding and be
|
||||
Runtime backend support is command-specific:
|
||||
|
||||
- `run`, `validate --config`, and `inspect --config` execute `local`, `ssh`, and `s3` sources.
|
||||
- `run` executes `local`, `ssh`, and `s3` destinations.
|
||||
- `run`, `reconcile-state`, and `prune` execute `local`, `ssh`, and `s3` destinations.
|
||||
- `serve` uses `http_upload` sources through the HTTP upload API and publishes to configured `local`, `ssh`, and `s3` destinations.
|
||||
- `http_upload` is valid only as a source backend.
|
||||
|
||||
@@ -35,7 +35,7 @@ pipelines:
|
||||
path: /srv/reports/archive
|
||||
```
|
||||
|
||||
This config publishes source files only. It uses default validation, destination path mapping, publish, transfer, and HTTP server values.
|
||||
This config publishes source files only. It uses default validation, destination path mapping, publish, state, reconciliation, retention, transfer, and HTTP server values.
|
||||
|
||||
## Production-Oriented Local Config
|
||||
|
||||
@@ -66,6 +66,8 @@ pipelines:
|
||||
html: false
|
||||
path_mapping:
|
||||
mode: preserve_relative
|
||||
reconciliation:
|
||||
mode: replace
|
||||
transfer:
|
||||
on_destination_same: skip
|
||||
on_destination_older: replace
|
||||
@@ -86,11 +88,15 @@ server:
|
||||
queue_size: 16
|
||||
max_concurrency: 1
|
||||
retention: 24h
|
||||
upload_tokens:
|
||||
- id: weather-reporter
|
||||
token_env: WEATHER_UPLOAD_TOKEN
|
||||
allow_pipelines:
|
||||
- weather-daily
|
||||
pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
staging_path: /var/spool/distributor/weather-daily
|
||||
max_upload_size: 20MB
|
||||
destinations:
|
||||
@@ -99,9 +105,9 @@ pipelines:
|
||||
path: /srv/reports/archive
|
||||
```
|
||||
|
||||
`token_env` is required for `http_upload` sources. `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`. `max_upload_size` defaults to `server.http.max_upload_size`.
|
||||
`upload_tokens` is required when any pipeline source uses `http_upload`. Each token record resolves its bearer token value from the process environment or `secrets.directory`. `allow_pipelines` lists configured upload pipeline ids that the token may submit to. One token may authorize multiple upload pipelines, and multiple tokens may authorize the same upload pipeline.
|
||||
|
||||
`serve` maps each resolved bearer token to exactly one `http_upload` pipeline. Startup fails when a token is missing, empty, or duplicates another upload pipeline token.
|
||||
For `http_upload` sources, `staging_path` defaults to `<server.http.staging_root>/<pipeline id>`. `max_upload_size` defaults to `server.http.max_upload_size`.
|
||||
|
||||
## Top-Level Fields
|
||||
|
||||
@@ -124,6 +130,18 @@ Numeric server values and durations must be greater than zero after defaults are
|
||||
|
||||
See [Secrets](#secrets) for resolution rules.
|
||||
|
||||
### `upload_tokens`
|
||||
|
||||
`upload_tokens` configures bearer tokens for `distributor serve`. It is required when any pipeline source backend is `http_upload` and is invalid when no upload pipelines are configured.
|
||||
|
||||
Each token has:
|
||||
|
||||
- `id`: required unique slug-like identifier for the token record. It must start with a letter or number and may contain letters, numbers, `.`, `_`, and `-`.
|
||||
- `token_env`: required environment variable or secret-file name containing the bearer token value.
|
||||
- `allow_pipelines`: required non-empty list of configured pipeline ids whose source backend is `http_upload`.
|
||||
|
||||
Token values must resolve to non-empty strings and must be unique across token records. Every configured upload pipeline must be allowed by at least one token.
|
||||
|
||||
### `pipelines`
|
||||
|
||||
`pipelines` is required and must contain at least one pipeline.
|
||||
@@ -216,13 +234,11 @@ HTTP upload backends are valid only as pipeline sources and are served by `distr
|
||||
|
||||
```yaml
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
staging_path: /var/spool/distributor/weather-daily
|
||||
max_upload_size: 20MB
|
||||
```
|
||||
|
||||
- `backend`: required value `http_upload`.
|
||||
- `token_env`: required environment variable or secret-file name containing the bearer token.
|
||||
- `staging_path`: optional staging path. Default: `<server.http.staging_root>/<pipeline id>`.
|
||||
- `max_upload_size`: optional per-source upload limit. Default: `server.http.max_upload_size`.
|
||||
|
||||
@@ -239,7 +255,7 @@ Source bundle digest mismatches fail validation before destination writes occur.
|
||||
|
||||
## Destination Fields
|
||||
|
||||
Each destination embeds a backend config at the destination level and may also configure publishing, transforms, path mapping, links, and transfer behavior.
|
||||
Each destination embeds a backend config at the destination level and may also configure publishing, transforms, path mapping, links, state, reconciliation, retention, and transfer behavior.
|
||||
|
||||
```yaml
|
||||
destinations:
|
||||
@@ -251,6 +267,10 @@ destinations:
|
||||
html: false
|
||||
path_mapping:
|
||||
mode: preserve_relative
|
||||
state:
|
||||
mode: single_owner
|
||||
reconciliation:
|
||||
mode: replace
|
||||
transfer:
|
||||
on_destination_same: skip
|
||||
on_destination_older: replace
|
||||
@@ -264,7 +284,10 @@ destinations:
|
||||
- `transform`: required only when publishing generated HTML.
|
||||
- `path_mapping`: optional destination path mapping policy.
|
||||
- `links`: optional public URL metadata policy.
|
||||
- `transfer`: optional destination reconciliation policy.
|
||||
- `state`: optional destination state ownership policy.
|
||||
- `reconciliation`: optional managed-output reconciliation policy.
|
||||
- `retention`: optional managed-output retention policy.
|
||||
- `transfer`: optional destination comparison action policy.
|
||||
|
||||
Destination ids must be unique within a pipeline.
|
||||
|
||||
@@ -340,6 +363,112 @@ Primary URL policies:
|
||||
|
||||
If no output matches the primary policy, per-output URLs may still be recorded and the top-level primary URL is omitted.
|
||||
|
||||
## Destination State Policy
|
||||
|
||||
```yaml
|
||||
state:
|
||||
mode: single_owner
|
||||
```
|
||||
|
||||
- `state.mode`: optional. Accepted values are `single_owner` and `shared_root`; default is `single_owner`.
|
||||
|
||||
`single_owner` state records one pipeline/destination owner for each destination bundle path and is the default state mode.
|
||||
|
||||
`shared_root` records multiple pipeline/destination owners in one destination root. Publish execution preserves unrelated owners, rejects path ownership conflicts, and writes shared-root destination state.
|
||||
|
||||
Use `shared_root` when multiple configured destinations intentionally write disjoint output paths into the same backend root:
|
||||
|
||||
```yaml
|
||||
pipelines:
|
||||
- id: reports-source
|
||||
source:
|
||||
backend: local
|
||||
path: /var/spool/distributor/reports
|
||||
destinations:
|
||||
- id: shared-root
|
||||
backend: local
|
||||
path: /srv/reports/shared
|
||||
state:
|
||||
mode: shared_root
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
- id: reports-html
|
||||
source:
|
||||
backend: local
|
||||
path: /var/spool/distributor/reports
|
||||
destinations:
|
||||
- id: shared-root
|
||||
backend: local
|
||||
path: /srv/reports/shared
|
||||
state:
|
||||
mode: shared_root
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
```
|
||||
|
||||
Every output path in a shared root belongs to exactly one `pipeline_id` and `destination_id`. A different owner planning the same path fails as a conflict.
|
||||
|
||||
## Reconciliation Policy
|
||||
|
||||
```yaml
|
||||
reconciliation:
|
||||
mode: replace
|
||||
```
|
||||
|
||||
- `reconciliation.mode`: optional. Accepted values are `replace` and `merge`; default is `replace`.
|
||||
|
||||
Reconciliation controls how a destination with older managed state is updated after transfer policy selects `replace_older`.
|
||||
|
||||
`replace` deletes the prior managed outputs recorded in `.distributor.json`, deletes the state file, verifies the destination bundle path is empty, then writes only the newly planned outputs and a new state file. This is the default and is appropriate when each publication should exactly match the current publish and transform policy.
|
||||
|
||||
`merge` keeps prior managed outputs that are not produced by the new plan. Planned paths already recorded in existing state may be overwritten; planned paths that already exist in storage but are not recorded as managed fail as unmanaged collisions. The resulting state file records the cumulative managed output set.
|
||||
|
||||
Example merge destination:
|
||||
|
||||
```yaml
|
||||
destinations:
|
||||
- id: static-site
|
||||
backend: local
|
||||
path: /srv/reports/site
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
reconciliation:
|
||||
mode: merge
|
||||
```
|
||||
|
||||
`links.primary_url` is selected from the newly planned outputs for the current run. Retained outputs keep their prior output metadata and timestamps.
|
||||
|
||||
## Retention Policy
|
||||
|
||||
```yaml
|
||||
retention:
|
||||
prune:
|
||||
enabled: false
|
||||
older_than: 168h
|
||||
keep_latest: 3
|
||||
```
|
||||
|
||||
- `retention.prune.enabled`: optional boolean. Default is `false`.
|
||||
- `retention.prune.older_than`: optional duration. When pruning is enabled, outputs older than this duration are eligible for pruning.
|
||||
- `retention.prune.keep_latest`: optional non-negative integer. When pruning is enabled, this many newest managed outputs are preserved before age-based pruning is considered.
|
||||
|
||||
When `retention.prune.enabled` is `true`, at least one of `older_than` or `keep_latest` is required. `older_than` must be greater than zero, and `keep_latest` must be zero or greater.
|
||||
|
||||
Pruning uses managed output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs.
|
||||
|
||||
The `prune` command is owner-scoped for shared-root state. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`.
|
||||
|
||||
## Transfer Policy
|
||||
|
||||
```yaml
|
||||
@@ -394,6 +523,9 @@ Defaults are applied after YAML decoding and before validation:
|
||||
- `transform.markdown_to_html.mode: sidecar` when a Markdown transform block is present and mode is omitted
|
||||
- `path_mapping.mode: preserve_relative`
|
||||
- `links.primary: auto` when a `links` block is present and `primary` is omitted
|
||||
- `state.mode: single_owner`
|
||||
- `reconciliation.mode: replace`
|
||||
- `retention.prune.enabled: false`
|
||||
- `transfer.on_destination_same: skip`
|
||||
- `transfer.on_destination_older: replace`
|
||||
- `transfer.on_destination_newer: skip`
|
||||
@@ -416,7 +548,7 @@ Fields resolved through this resolver:
|
||||
|
||||
- `credentials.access_key_id_env`
|
||||
- `credentials.secret_access_key_env`
|
||||
- `source.token_env` for `http_upload` sources
|
||||
- `upload_tokens[].token_env`
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
@@ -430,6 +562,8 @@ Local examples:
|
||||
- `local-index.yml`: local `index.html` publication.
|
||||
- `fan-out.yml`: local fan-out publication to source and HTML destinations.
|
||||
- `archive-and-latest.yml`: local archive plus fixed latest publication.
|
||||
- `merge-reconciliation.yml`: local HTML publication using merge reconciliation.
|
||||
- `shared-root.yml`: two local pipelines publishing disjoint outputs into one shared destination root.
|
||||
- `http-upload-local.yml`: local HTTP upload server config; requires `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN` in the process environment or as a secret-file name before running `serve`.
|
||||
|
||||
Environment-gated remote examples:
|
||||
|
||||
136
docs/consumers/api.md
Normal file
136
docs/consumers/api.md
Normal file
@@ -0,0 +1,136 @@
|
||||
# Upstream Producer Integration
|
||||
|
||||
Audience: developers and LLM coding agents adding `distributor` support to an upstream Go producer application.
|
||||
|
||||
This document is the copyable implementation guide for submitting producer outputs to a `distributor` pipeline whose source backend is `http_upload`.
|
||||
|
||||
## Required Inputs
|
||||
|
||||
The upstream application needs these values from deployment or operator configuration:
|
||||
|
||||
- distributor endpoint: the HTTP server base URL, such as `https://distributor.example.com`;
|
||||
- upload token: bearer token that authenticates the producer;
|
||||
- pipeline id: configured `http_upload` pipeline that should process this upload;
|
||||
- generated files: regular local files to include in the source bundle;
|
||||
- bundle id: stable identifier for the logical report stream or artifact;
|
||||
- idempotency key: unique key for one producer run, reused only when retrying that same run.
|
||||
|
||||
Do not put destination routing, public URLs, transform settings, or credentials in the source manifest. Those belong in the `distributor` pipeline configuration.
|
||||
|
||||
The token, pipeline id, bundle id, and idempotency key have different jobs. The token authenticates the producer. The pipeline id selects the configured distributor workflow, including destinations and publishing policy. The bundle id tells `distributor` whether a new upload is a newer version of the same source; keep it stable across runs that should replace the same managed destination artifact. The idempotency key tells `distributor` whether an upload request is a retry; change it for each distinct producer run so new content is enqueued.
|
||||
|
||||
## Recommended Workflow
|
||||
|
||||
Use `gitea.maximumdirect.net/eric/distributor/pkg/upload`.
|
||||
|
||||
For most producers, use `UploadFiles`. It accepts producer-generated files, builds a temporary valid source bundle with `pkg/bundle`, uploads a gzip-compressed tar archive, and removes temporary files when the call returns.
|
||||
|
||||
Use `UploadBundle` only when the producer already assembled a complete bundle directory containing `manifest.json`.
|
||||
|
||||
Add the dependency from the upstream application:
|
||||
|
||||
```sh
|
||||
go get gitea.maximumdirect.net/eric/distributor
|
||||
```
|
||||
|
||||
## Minimal Go Example
|
||||
|
||||
```go
|
||||
package reports
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/pkg/upload"
|
||||
)
|
||||
|
||||
func SubmitReport(reportPath, summaryPath string) error {
|
||||
endpoint := os.Getenv("DISTRIBUTOR_UPLOAD_ENDPOINT")
|
||||
token := os.Getenv("DISTRIBUTOR_UPLOAD_TOKEN")
|
||||
if endpoint == "" || token == "" {
|
||||
return fmt.Errorf("distributor endpoint and token are required")
|
||||
}
|
||||
|
||||
pipelineID := "weather-hourly"
|
||||
reportID := "weather.hourly.brentwood"
|
||||
runID := time.Now().UTC().Format("20060102T150405.000000000Z")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
client, err := upload.NewClient(upload.ClientOptions{
|
||||
Endpoint: endpoint,
|
||||
Token: token,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
|
||||
PipelineID: pipelineID,
|
||||
ID: reportID,
|
||||
IdempotencyKey: reportID + "." + runID,
|
||||
Files: []bundle.BundleFile{
|
||||
{SourcePath: reportPath, Path: "report.md"},
|
||||
{SourcePath: summaryPath, Path: "summary.txt"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
var conflict *upload.IdempotencyConflictError
|
||||
if errors.As(err, &conflict) {
|
||||
return fmt.Errorf("idempotency key was reused for different bundle content: %w", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("distributor accepted run %s\n", result.RunID)
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
## Producer Responsibilities
|
||||
|
||||
- Use a stable bundle id for the logical producer output that should replace the same destination artifact, such as `weather.hourly.brentwood`.
|
||||
- Set `PipelineID` to the configured upload pipeline that should process the bundle.
|
||||
- Do not include per-run timestamps, random values, or job ids in the bundle id unless each run should be treated as a different source.
|
||||
- Use an idempotency key that changes for every distinct producer run, such as `<bundle-id>.<run-id>`.
|
||||
- Reuse the same idempotency key only when retrying the exact same producer run with the same source manifest.
|
||||
- Map each generated file to a clean slash-separated bundle path, such as `report.md` or `assets/chart.png`.
|
||||
- Include only regular files. Symlinks, directories as files, devices, FIFOs, and sockets are rejected.
|
||||
- Keep file contents stable after upload inputs are selected. Bundle digests are calculated from file bytes.
|
||||
- Treat upload success as admission only. `UploadFiles` and `UploadBundle` return after the server accepts and validates the upload, not after all destinations publish.
|
||||
|
||||
Valid bundle paths are relative slash paths. They must not be empty, absolute, contain backslashes, contain `.` or `..` path segments, contain empty path segments, or use reserved basenames `manifest.json` or `.distributor.json`.
|
||||
|
||||
## Idempotency And Status
|
||||
|
||||
`pkg/upload` sends `Idempotency-Key` on every upload. If the caller omits one, the package generates a random key for that call and reuses it for in-process retries. That is enough for transient network retry within one process, but it does not give cross-process retry identity.
|
||||
|
||||
For producer jobs that may retry after process restart, supply a key derived from the producer run, such as `<bundle-id>.<run-id>`. Reusing the same key with the same token, pipeline id, and normalized source manifest returns the original accepted run. Reusing the same key with different source content in that scope returns a conflict. Reusing one key across multiple distinct report generations prevents those generations from being treated as new uploads.
|
||||
|
||||
`Status` polls `/runs/<run-id>` while the distributor server retains the in-memory status record. Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Completed records expire according to the server's `server.http.retention` setting, and server restart clears status and idempotency records.
|
||||
|
||||
Optional status check:
|
||||
|
||||
```go
|
||||
status, err := client.Status(ctx, result.RunID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if status.Status == "failed" {
|
||||
return fmt.Errorf("distributor run failed: %s", status.Error)
|
||||
}
|
||||
```
|
||||
|
||||
## References
|
||||
|
||||
In the `distributor` source tree:
|
||||
|
||||
- `docs/consumers/pkg-upload.md`: Go upload package workflow.
|
||||
- `docs/consumers/pkg-bundle.md`: Go bundle package workflow.
|
||||
- `docs/integrations/http-upload.md`: canonical HTTP upload wire contract.
|
||||
- `docs/integrations/source-bundle.md`: canonical source bundle file-format contract.
|
||||
90
docs/consumers/pkg-bundle.md
Normal file
90
docs/consumers/pkg-bundle.md
Normal file
@@ -0,0 +1,90 @@
|
||||
# `pkg/bundle`
|
||||
|
||||
Audience: upstream Go producer developers and LLM coding agents using `distributor` source bundle helpers.
|
||||
|
||||
Import path:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
```
|
||||
|
||||
`pkg/bundle` builds, writes, parses, and validates local source bundles. Use it directly when a producer writes bundles for `distributor` to discover, or when a producer wants to assemble and validate a bundle before using another transport.
|
||||
|
||||
The canonical source bundle file-format contract is [Source Bundle Contract](../integrations/source-bundle.md).
|
||||
|
||||
## Preferred Complete-Bundle Workflow
|
||||
|
||||
Use `WriteBundle` when producer-generated files live outside the final bundle root.
|
||||
|
||||
```go
|
||||
manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{
|
||||
Root: "/var/spool/distributor/weather/hourly-2026-06-07T15",
|
||||
ID: "weather.hourly.brentwood",
|
||||
Files: []bundle.BundleFile{
|
||||
{SourcePath: "/tmp/weather/report.md", Path: "report.md"},
|
||||
{SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = manifest
|
||||
```
|
||||
|
||||
`WriteBundle` copies each source file into a staged bundle root, writes `manifest.json`, validates the staged bundle, and promotes it into place. Set `Overwrite: true` only when the producer intentionally replaces an existing bundle root.
|
||||
|
||||
## Existing Bundle Root Workflow
|
||||
|
||||
Use `BuildManifest` and `WriteManifest` when files are already staged under the final bundle root.
|
||||
|
||||
```go
|
||||
root := "/var/spool/distributor/weather/hourly-2026-06-07T15"
|
||||
manifest, err := bundle.BuildManifest(bundle.BuildOptions{
|
||||
Root: root,
|
||||
ID: "weather.hourly.brentwood",
|
||||
Files: []string{"report.md", "summary.txt"},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bundle.WriteManifest(root, manifest, bundle.WriteManifestOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bundle.ValidateBundle(root, manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
Use `Scan: true` instead of `Files` only when every valid regular file under the root should be included. Scan mode includes dotfiles, skips reserved metadata files, rejects symlinks, and sorts paths lexically.
|
||||
|
||||
## Paths And Ordering
|
||||
|
||||
Bundle paths are slash-separated paths relative to the bundle root.
|
||||
|
||||
Invalid paths include:
|
||||
|
||||
- empty paths;
|
||||
- absolute paths;
|
||||
- paths containing backslashes;
|
||||
- `.` or `..` path segments;
|
||||
- empty path segments;
|
||||
- any basename of `manifest.json` or `.distributor.json`.
|
||||
|
||||
Explicit file lists preserve caller order. File order is part of the bundle digest, so producers should choose it deliberately and keep it stable.
|
||||
|
||||
The manifest `ID` is the logical source identity used by `distributor` destination comparison. Keep it stable for runs that should replace the same managed destination artifact. If every run uses a different manifest `ID`, `distributor` treats those runs as different sources and may report a destination conflict instead of replacing older output.
|
||||
|
||||
## Validation And Digest Helpers
|
||||
|
||||
Use `ValidateBundle` before handing an existing local bundle to another process. It verifies manifest semantics, file existence, regular-file type, file size, per-file SHA-256 digests, and bundle digest.
|
||||
|
||||
Useful helpers:
|
||||
|
||||
- `LoadManifest`: read `manifest.json` from a bundle root.
|
||||
- `ParseManifest` and `MarshalManifest`: parse or write manifest bytes.
|
||||
- `ValidateManifest`: validate manifest-only semantics.
|
||||
- `FileDigest`, `BundleDigest`, and `ValidateDigest`: digest helpers for diagnostics and tests.
|
||||
|
||||
## Boundaries
|
||||
|
||||
`pkg/bundle` does not upload bundles, publish destinations, transform Markdown, select pipelines, configure credentials, or write destination state. Those concerns belong to `pkg/upload` or the `distributor` application.
|
||||
122
docs/consumers/pkg-upload.md
Normal file
122
docs/consumers/pkg-upload.md
Normal file
@@ -0,0 +1,122 @@
|
||||
# `pkg/upload`
|
||||
|
||||
Audience: upstream Go producer developers and LLM coding agents submitting bundles to `distributor serve`.
|
||||
|
||||
Import path:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/distributor/pkg/upload"
|
||||
```
|
||||
|
||||
`pkg/upload` is the producer-facing HTTP upload client. It builds on `pkg/bundle`, packages valid source bundles as gzip-compressed tar archives, sends bearer authentication, routes uploads to a configured pipeline, includes idempotency keys, and exposes a status polling helper.
|
||||
|
||||
`UploadFiles` examples also use:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
|
||||
```
|
||||
|
||||
The canonical HTTP wire contract is [HTTP Upload API Contract](../integrations/http-upload.md).
|
||||
|
||||
## Client Construction
|
||||
|
||||
```go
|
||||
client, err := upload.NewClient(upload.ClientOptions{
|
||||
Endpoint: "https://distributor.example.com",
|
||||
Token: token,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
`Endpoint` is the distributor server base URL. The client derives `/v1/pipelines/<pipeline-id>/upload` and `/runs/<run-id>`. `Token` is required and is sent as `Authorization: Bearer <token>`. Token values are redacted from client errors.
|
||||
|
||||
`HTTPClient` and `Retry` are optional. Defaults use a 30 second HTTP timeout and safe retry settings.
|
||||
|
||||
## Upload Producer Files
|
||||
|
||||
Use `UploadFiles` when the producer has generated output files but has not assembled a bundle directory.
|
||||
|
||||
```go
|
||||
result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
|
||||
PipelineID: "weather-hourly",
|
||||
ID: "weather.hourly.brentwood",
|
||||
IdempotencyKey: "weather.hourly.brentwood.20260607T150000Z",
|
||||
Files: []bundle.BundleFile{
|
||||
{SourcePath: "/tmp/weather/report.md", Path: "report.md"},
|
||||
{SourcePath: "/tmp/weather/summary.txt", Path: "summary.txt"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = result.RunID
|
||||
```
|
||||
|
||||
`PipelineID` is required and selects the configured distributor workflow for this upload. `ID` is the source manifest id and identifies the logical artifact inside that workflow. `UploadFiles` creates a temporary bundle, writes and validates a manifest, uploads the archive, and removes temporary files when the call returns. It does not write into producer source directories.
|
||||
|
||||
## Upload An Existing Bundle
|
||||
|
||||
Use `UploadBundle` when the producer already has a complete local bundle root containing `manifest.json`.
|
||||
|
||||
```go
|
||||
result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
|
||||
PipelineID: "weather-hourly",
|
||||
Root: "/var/spool/weather/hourly-2026-06-07T15",
|
||||
IdempotencyKey: "weather.hourly.brentwood.20260607T150000Z",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = result.RunID
|
||||
```
|
||||
|
||||
`PipelineID` is required for existing bundles too. `UploadBundle` validates the local bundle by default and uploads only `manifest.json` plus manifest-listed files. Unlisted files are not uploaded.
|
||||
|
||||
## Result And Status
|
||||
|
||||
Upload success means the server returned `202 Accepted` after staging and validating the upload. It does not mean all configured destinations have published.
|
||||
|
||||
Poll status while the server retains the in-memory run record:
|
||||
|
||||
```go
|
||||
status, err := client.Status(ctx, result.RunID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if status.Status == "failed" {
|
||||
return fmt.Errorf("distributor run failed: %s", status.Error)
|
||||
}
|
||||
```
|
||||
|
||||
Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Completed records expire according to `server.http.retention`; server restart clears run status and idempotency records.
|
||||
|
||||
## Idempotency And Retry
|
||||
|
||||
Every upload request includes `Idempotency-Key`.
|
||||
|
||||
If `IdempotencyKey` is omitted, the client generates a random 128-bit lowercase hexadecimal key for that upload operation and reuses it for retries within the same call. For cross-process retry safety, producers should pass a key derived from the producer run, such as `<bundle-id>.<run-id>`.
|
||||
|
||||
Do not reuse the same idempotency key for multiple distinct report generations. Reuse it only when retrying the exact same run with the same token, pipeline id, and source manifest. A repeated key with the same manifest in that scope returns the original accepted run instead of enqueueing another run; a repeated key with different content returns an idempotency conflict.
|
||||
|
||||
The client retries only safe cases:
|
||||
|
||||
- `503 Service Unavailable`;
|
||||
- temporary network errors;
|
||||
- ambiguous mid-upload failures.
|
||||
|
||||
It does not retry after `202 Accepted` and does not retry `400`, `401`, `403`, `404`, `409`, `413`, or `415`.
|
||||
|
||||
Detect conflicting key reuse with `errors.As`:
|
||||
|
||||
```go
|
||||
var conflict *upload.IdempotencyConflictError
|
||||
if errors.As(err, &conflict) {
|
||||
return fmt.Errorf("idempotency key was reused for different bundle content: %w", err)
|
||||
}
|
||||
```
|
||||
|
||||
## Boundaries
|
||||
|
||||
`pkg/upload` does not configure server pipelines, choose destinations, wait for publication completion automatically, persist client queues, provide durable idempotency across server restarts, or expose destination state. It submits complete source bundles to the configured HTTP upload API.
|
||||
@@ -4,17 +4,25 @@ Audience: operators, integrators, and maintainers who inspect or reason about de
|
||||
|
||||
Each managed destination bundle path contains `.distributor.json`. This file is the destination sentinel and state record used for comparison, skip, replacement, and recovery decisions.
|
||||
|
||||
## State Schema
|
||||
## Single-Owner State Schema
|
||||
|
||||
Current schema version: `1`.
|
||||
State written by `run` for `state.mode: single_owner` uses schema version `2`.
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"schema_version": 2,
|
||||
"distributor_version": "dev",
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"published_at": "2026-06-04T12:00:00Z",
|
||||
"created_at": "2026-06-04T12:00:00Z",
|
||||
"updated_at": "2026-06-04T12:00:00Z",
|
||||
"state": {
|
||||
"mode": "single_owner"
|
||||
},
|
||||
"reconciliation": {
|
||||
"mode": "replace"
|
||||
},
|
||||
"source": {
|
||||
"manifest": {
|
||||
"schema_version": 1,
|
||||
@@ -37,7 +45,9 @@ Current schema version: `1`.
|
||||
"transform": "markdown_to_html",
|
||||
"url": "https://reports.example.com/archive/report.html",
|
||||
"sha256": "sha256:...",
|
||||
"size": 2345
|
||||
"size": 2345,
|
||||
"created_at": "2026-06-04T12:00:00Z",
|
||||
"updated_at": "2026-06-04T12:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -45,10 +55,14 @@ Current schema version: `1`.
|
||||
|
||||
Required fields:
|
||||
|
||||
- `schema_version`: must be `1`.
|
||||
- `schema_version`: must be `2` for newly written state.
|
||||
- `pipeline_id`: configured pipeline id that wrote the state.
|
||||
- `destination_id`: configured destination id that wrote the state.
|
||||
- `published_at`: RFC3339 publication timestamp.
|
||||
- `created_at`: RFC3339 timestamp for when this destination state record was first created.
|
||||
- `updated_at`: RFC3339 timestamp for the latest state update.
|
||||
- `state.mode`: must be `single_owner`.
|
||||
- `reconciliation.mode`: `replace` or `merge`.
|
||||
- `source.manifest`: embedded source bundle manifest.
|
||||
- `outputs`: output records array; empty is allowed, but the field is required.
|
||||
|
||||
@@ -68,15 +82,31 @@ Each output record has:
|
||||
- `url`: optional absolute HTTP or HTTPS URL for the output.
|
||||
- `sha256`: lowercase `sha256:<64 hex>` digest of the output bytes.
|
||||
- `size`: output byte size, zero or greater.
|
||||
- `created_at`: RFC3339 timestamp for when this output path was first recorded as managed.
|
||||
- `updated_at`: RFC3339 timestamp for when this output path was last written or updated in state.
|
||||
|
||||
Output paths must be unique and use clean relative slash-separated path rules.
|
||||
|
||||
For replacement updates, newly planned outputs are written into state. For merge updates, retained output records preserve both timestamps, overwritten managed output records preserve `created_at` and receive a new `updated_at`, and new output records receive the current publication time for both fields.
|
||||
|
||||
## Reconciliation Semantics
|
||||
|
||||
Destination reconciliation applies when destination state is older than the source and transfer policy permits replacement.
|
||||
|
||||
- `replace`: for single-owner state, delete managed output paths recorded in `outputs` plus `.distributor.json`, require the destination bundle path to be empty afterward, write the newly planned outputs, and write state whose `outputs` are exactly that new planned set. For shared-root state, delete only the current owner's omitted outputs and preserve unrelated owners.
|
||||
- `merge`: retain managed output paths omitted from the new plan, overwrite planned paths only when they are already recorded in existing state, fail when a newly planned path exists in storage but is not recorded as managed, and write state whose `outputs` are the cumulative managed set.
|
||||
|
||||
Top-level `links.primary_url` is selected from the newly planned outputs for the current publication. Retained outputs keep their existing per-output URL metadata.
|
||||
|
||||
If a merge publication fails after writing outputs, cleanup removes only newly created outputs from that failed attempt. Previously managed overwritten paths remain managed and are not removed by failed-attempt cleanup.
|
||||
|
||||
## Comparison Semantics
|
||||
|
||||
`distributor` compares the current source manifest to destination state before writing:
|
||||
|
||||
- No state and no content: publish new outputs.
|
||||
- No state and existing content: treat the destination as unmanaged.
|
||||
- Shared-root state without the current owner: publish new outputs for that owner if planned paths do not collide with other owners or unmanaged content.
|
||||
- Matching embedded source manifest: skip.
|
||||
- Same source id with older `created`: replace if policy allows.
|
||||
- Same source id with newer `created`: skip by default.
|
||||
@@ -84,7 +114,131 @@ Output paths must be unique and use clean relative slash-separated path rules.
|
||||
- Different source id, pipeline id, or destination id: conflict.
|
||||
- Invalid state JSON or invalid state fields: conflict.
|
||||
|
||||
Normal replacement deletes only managed output paths recorded in `outputs` plus `.distributor.json`. Forced replacement deletes the bounded destination bundle path.
|
||||
Normal single-owner replacement deletes only managed output paths recorded in `outputs` plus `.distributor.json`. Shared-root replacement deletes only omitted outputs for the current owner. Merge publication retains omitted managed outputs. Forced replacement deletes the bounded destination bundle path.
|
||||
|
||||
## State Repair Semantics
|
||||
|
||||
`distributor reconcile-state` can remove managed output records for files that no longer exist in destination storage. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||
|
||||
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. The command checks output paths recorded in `outputs`, reports missing managed outputs, reports unmanaged entries under the destination root, and removes missing output records from valid state unless `--dry-run` is set.
|
||||
|
||||
For shared-root state, repair is scoped to the selected owner by default. With `--all-owners`, it checks and repairs missing output records for every owner in the selected shared-root state file.
|
||||
|
||||
State repair does not validate output digests, delete destination files, adopt unmanaged entries, or rewrite invalid or mismatched state.
|
||||
|
||||
## Prune Semantics
|
||||
|
||||
`distributor prune` can delete managed outputs selected by the configured destination retention policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||
|
||||
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
|
||||
|
||||
Prune planning uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`.
|
||||
|
||||
## Compatibility
|
||||
|
||||
`distributor` can read schema version `1` destination state for compatibility. When v1 state is read, it is treated as single-owner state with `reconciliation.mode: replace`. Missing top-level `created_at` and `updated_at` are inferred from `published_at`, and missing per-output timestamps are also inferred from `published_at`.
|
||||
|
||||
Newly written single-owner destination state from publish execution uses schema version `2`.
|
||||
|
||||
## Shared-Root State Schema
|
||||
|
||||
State written by `run` for `state.mode: shared_root` uses schema version `3`.
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 3,
|
||||
"distributor_version": "dev",
|
||||
"created_at": "2026-06-04T12:00:00Z",
|
||||
"updated_at": "2026-06-04T12:10:00Z",
|
||||
"state": {
|
||||
"mode": "shared_root"
|
||||
},
|
||||
"owners": [
|
||||
{
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"reconciliation": {
|
||||
"mode": "merge"
|
||||
},
|
||||
"source": {
|
||||
"manifest": {
|
||||
"schema_version": 1,
|
||||
"id": "reports.example.2026-06-04",
|
||||
"digest": "sha256:...",
|
||||
"created": "2026-06-04T11:55:00Z",
|
||||
"files": [
|
||||
{"path": "report.md", "sha256": "sha256:...", "size": 1234}
|
||||
]
|
||||
}
|
||||
},
|
||||
"links": {
|
||||
"primary_url": "https://reports.example.com/archive/report.html"
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"path": "report.html",
|
||||
"kind": "generated",
|
||||
"source_path": "report.md",
|
||||
"transform": "markdown_to_html",
|
||||
"url": "https://reports.example.com/archive/report.html",
|
||||
"sha256": "sha256:...",
|
||||
"size": 2345,
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"source_id": "reports.example.2026-06-04",
|
||||
"source_digest": "sha256:...",
|
||||
"source_created": "2026-06-04T11:55:00Z",
|
||||
"created_at": "2026-06-04T12:00:00Z",
|
||||
"updated_at": "2026-06-04T12:10:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Shared-root required fields:
|
||||
|
||||
- `schema_version`: must be `3`.
|
||||
- `created_at`: RFC3339 timestamp for when this shared-root state record was first created.
|
||||
- `updated_at`: RFC3339 timestamp for the latest shared-root state update.
|
||||
- `state.mode`: must be `shared_root`.
|
||||
- `owners`: owner records keyed by `pipeline_id` and `destination_id`; each owner records its latest source manifest and reconciliation mode.
|
||||
- `outputs`: output records for every managed path under the shared destination root.
|
||||
|
||||
Shared-root optional fields:
|
||||
|
||||
- `distributor_version`: application version string when available.
|
||||
- `owners[].links.primary_url`: absolute HTTP or HTTPS URL selected by that owner's destination link policy.
|
||||
|
||||
Shared-root output records carry the same `path`, `kind`, `source_path`, `transform`, `url`, `sha256`, `size`, `created_at`, and `updated_at` fields as single-owner outputs. They also include the owner `pipeline_id` and `destination_id`, plus compact source identity fields `source_id`, `source_digest`, and `source_created`.
|
||||
|
||||
## Shared-Root Ownership
|
||||
|
||||
Shared-root state is owner-scoped by `pipeline_id` and `destination_id`.
|
||||
|
||||
- One output path may be owned by only one owner.
|
||||
- The same owner may overwrite its own managed paths.
|
||||
- A different owner planning an already owned path fails as a conflict.
|
||||
- A planned path that exists in storage but is not recorded in state fails as unmanaged content unless forced replacement is explicitly selected.
|
||||
|
||||
When an owner publishes, unrelated owner records and output records are preserved. The publishing owner's record is updated with the latest source manifest, reconciliation mode, and latest primary URL when present.
|
||||
|
||||
## Shared-Root Timestamps
|
||||
|
||||
For shared-root state:
|
||||
|
||||
- top-level `created_at` remains the original shared-root state creation time;
|
||||
- top-level `updated_at` changes after a successful state write;
|
||||
- output `created_at` remains stable for an existing managed path;
|
||||
- output `updated_at` changes only when that path is rewritten;
|
||||
- newly managed output paths receive the publication time for both output timestamps.
|
||||
|
||||
## Shared-Root Migration
|
||||
|
||||
If `state.mode: shared_root` is configured and existing state is a compatible single-owner `.distributor.json` for the same pipeline id and destination id, the next successful publish writes schema version `3` shared-root state for that owner.
|
||||
|
||||
If existing single-owner state belongs to a different pipeline or destination, publish fails as a conflict. `distributor` does not implicitly take over unrelated state or unmanaged files.
|
||||
|
||||
## Boundaries
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Audience: producers, operators, and maintainers integrating with `distributor serve`.
|
||||
|
||||
`distributor serve` exposes a local HTTP upload API for pipelines whose source backend is `http_upload`. Each bearer token maps to exactly one configured pipeline.
|
||||
`distributor serve` exposes a local HTTP upload API for pipelines whose source backend is `http_upload`. Bearer tokens authenticate producers, and the upload path selects the configured pipeline. The selected token must be allowed for the requested pipeline.
|
||||
|
||||
## Authentication
|
||||
|
||||
@@ -12,9 +12,9 @@ Uploads authenticate with:
|
||||
Authorization: Bearer <token>
|
||||
```
|
||||
|
||||
Token values are resolved from the configured `source.token_env` through the process environment or `secrets.directory`. Tokens are not configured as YAML literal values.
|
||||
Token values are resolved from top-level `upload_tokens` records through the process environment or `secrets.directory`. Tokens are not configured as YAML literal values.
|
||||
|
||||
Requests that include `pipeline` or `pipeline_id` query parameters are rejected. The bearer token selects the pipeline.
|
||||
Requests that include `pipeline` or `pipeline_id` query parameters are rejected. Use the pipeline id in the upload path.
|
||||
|
||||
## Endpoints
|
||||
|
||||
@@ -26,7 +26,7 @@ Returns `200 OK` when the server is running:
|
||||
{"status":"ok"}
|
||||
```
|
||||
|
||||
### `POST /upload`
|
||||
### `POST /v1/pipelines/{pipeline_id}/upload`
|
||||
|
||||
Accepts one source bundle archive and returns after the archive is staged and validated.
|
||||
|
||||
@@ -36,7 +36,7 @@ Producers may include:
|
||||
Idempotency-Key: <key>
|
||||
```
|
||||
|
||||
Idempotency keys are scoped to the authenticated pipeline selected by the bearer token. Valid keys are non-empty ASCII strings up to 128 bytes using letters, digits, `.`, `_`, `-`, and `:`. Invalid keys return `400`.
|
||||
`pipeline_id` must name a configured pipeline whose source backend is `http_upload`, and the authenticated token must allow that pipeline. Idempotency keys are scoped to token id, pipeline id, and key. Valid keys are non-empty ASCII strings up to 128 bytes using letters, digits, `.`, `_`, `-`, and `:`. Invalid keys return `400`.
|
||||
|
||||
Accepted content types:
|
||||
|
||||
@@ -54,6 +54,8 @@ Common error responses:
|
||||
|
||||
- `400`: pipeline query supplied, invalid idempotency key, archive rejected, malformed archive, or invalid staged source bundle.
|
||||
- `401`: missing, empty, or unknown bearer token.
|
||||
- `403`: bearer token is valid but is not allowed for the requested pipeline.
|
||||
- `404`: upload path is unknown or the requested upload pipeline is not configured.
|
||||
- `409`: repeated idempotency key conflicts with another source manifest, or the same key is already being staged.
|
||||
- `413`: upload body exceeds the selected pipeline size limit.
|
||||
- `415`: unsupported content type.
|
||||
@@ -71,7 +73,7 @@ Retryable idempotency conflicts include:
|
||||
{"error":"upload idempotency key is already being processed","retryable":true}
|
||||
```
|
||||
|
||||
When `Idempotency-Key` is omitted, upload admission preserves the raw HTTP behavior: every valid accepted upload receives its own run id. When a key is supplied, the server records the accepted run after archive staging and source bundle validation succeed. Reusing the same key for the same authenticated pipeline and the same normalized source manifest returns the original `202 Accepted` response and does not enqueue another run. Reusing the same key for a different normalized source manifest returns `409 Conflict`.
|
||||
When `Idempotency-Key` is omitted, upload admission preserves the raw HTTP behavior: every valid accepted upload receives its own run id. When a key is supplied, the server records the accepted run after archive staging and source bundle validation succeed. Reusing the same key for the same token id, pipeline id, and normalized source manifest returns the original `202 Accepted` response and does not enqueue another run. Reusing the same key for a different normalized source manifest within that scope returns `409 Conflict`. Producers should use a fresh key for each distinct producer run and reuse a key only for retries of that same run.
|
||||
|
||||
### `GET /runs/<run-id>`
|
||||
|
||||
@@ -108,7 +110,7 @@ The uploaded archive size and extracted bundle size are bounded by the selected
|
||||
|
||||
## Go Producer Helper
|
||||
|
||||
Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/upload` to build or validate source bundles, package them as gzip-compressed tar archives, and submit them to this API:
|
||||
Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/upload` to build or validate source bundles, package them as gzip-compressed tar archives, and submit them to this API. See [Upstream Producer Integration](../consumers/api.md) for the copyable upstream implementation guide and [`pkg/upload`](../consumers/pkg-upload.md) for package-specific workflow guidance.
|
||||
|
||||
```go
|
||||
client, err := upload.NewClient(upload.ClientOptions{
|
||||
@@ -119,14 +121,15 @@ if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
|
||||
PipelineID: "reports",
|
||||
Root: "examples/source-bundle",
|
||||
IdempotencyKey: "reports.example.20260604T120000Z",
|
||||
})
|
||||
```
|
||||
|
||||
`Endpoint` is the server base URL; the package derives `/upload` and `/runs/<run-id>`. `UploadBundle` validates a local bundle by default and uploads only `manifest.json` plus manifest-listed files. `UploadFiles` creates a temporary bundle from explicit `bundle.BundleFile` values before uploading. When `IdempotencyKey` is omitted, the package generates one random 128-bit lowercase hex key for the upload operation and reuses it across retries.
|
||||
`Endpoint` is the server base URL; the package derives `/v1/pipelines/<pipeline-id>/upload` and `/runs/<run-id>`. `PipelineID` is required and selects the configured distributor workflow. `UploadBundle` validates a local bundle by default and uploads only `manifest.json` plus manifest-listed files. `UploadFiles` creates a temporary bundle from explicit `bundle.BundleFile` values before uploading. When `IdempotencyKey` is omitted, the package generates one random 128-bit lowercase hex key for the upload operation and reuses it across retries.
|
||||
|
||||
The helper retries only safe cases: `503 Service Unavailable`, temporary network errors, and ambiguous mid-upload failures. It does not retry after `202 Accepted` and does not retry `400`, `401`, `409`, `413`, or `415`. Bearer token values are redacted from returned errors.
|
||||
The helper retries only safe cases: `503 Service Unavailable`, temporary network errors, and ambiguous mid-upload failures. It does not retry after `202 Accepted` and does not retry `400`, `401`, `403`, `404`, `409`, `413`, or `415`. Bearer token values are redacted from returned errors.
|
||||
|
||||
## Queue And Retention
|
||||
|
||||
@@ -138,7 +141,7 @@ Idempotency records are memory-only, expire with completed upload status records
|
||||
|
||||
## Boundaries
|
||||
|
||||
The HTTP API does not expose pipeline selection by request parameter, TLS, public routing policy, or durable status storage. Put public access controls, TLS termination, and rate limiting in deployment infrastructure.
|
||||
The HTTP API does not expose pipeline selection by query parameter, TLS, public routing policy, or durable status storage. Put public access controls, TLS termination, and rate limiting in deployment infrastructure.
|
||||
|
||||
## Tests
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ The adapter uses these S3 operations:
|
||||
|
||||
Writes buffer the input and set `ContentLength`. If no content type is supplied by the caller, the adapter infers a content type from the logical path.
|
||||
|
||||
Normal replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Forced replacement deletes objects under the bounded destination bundle prefix. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
|
||||
Normal replacement and failed-write cleanup delete only managed output objects plus `.distributor.json`. Merge publication retains omitted managed objects and may overwrite existing managed objects. Forced replacement deletes objects under the bounded destination bundle prefix. The backend does not manage bucket versioning, lifecycle rules, object lock, or delete markers.
|
||||
|
||||
## Error Mapping
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ Current schema version: `1`.
|
||||
Required manifest fields:
|
||||
|
||||
- `schema_version`: must be `1`.
|
||||
- `id`: non-empty bundle identifier.
|
||||
- `id`: non-empty bundle identifier. For replacement workflows, keep this stable for the logical source that should update the same managed destination artifact.
|
||||
- `digest`: lowercase `sha256:<64 hex>` digest of the ordered `files` list.
|
||||
- `created`: RFC3339 timestamp.
|
||||
- `files`: non-empty ordered list of file records.
|
||||
@@ -60,7 +60,7 @@ File order is significant. Explicit file lists preserve caller order. Scan mode
|
||||
|
||||
## Producer APIs
|
||||
|
||||
Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build and validate this contract:
|
||||
Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to build and validate this contract. See [`pkg/bundle`](../consumers/pkg-bundle.md) for producer workflow guidance.
|
||||
|
||||
- `BuildManifest`: builds a manifest from explicit file paths or scan mode.
|
||||
- `WriteManifest`: writes `manifest.json`, optionally replacing an existing manifest.
|
||||
@@ -68,7 +68,7 @@ Go producers can use `gitea.maximumdirect.net/eric/distributor/pkg/bundle` to bu
|
||||
- `LoadManifest`, `ParseManifest`, `ValidateManifest`, and `ValidateBundle`: parse and validate local bundles.
|
||||
- `FileDigest`, `BundleDigest`, and `ValidateDigest`: digest helpers.
|
||||
|
||||
Go producers that submit bundles to `distributor serve` can use `gitea.maximumdirect.net/eric/distributor/pkg/upload`. It builds on `pkg/bundle`, packages valid bundles as gzip-compressed tar uploads, sends bearer authentication, and includes idempotency keys for safe retry behavior. See [HTTP Upload API Contract](http-upload.md).
|
||||
Go producers that submit bundles to `distributor serve` can use `gitea.maximumdirect.net/eric/distributor/pkg/upload`. See [Upstream Producer Integration](../consumers/api.md) and [HTTP Upload API Contract](http-upload.md).
|
||||
|
||||
CLI producers can use:
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ The configured `path` is the backend root. All source discovery, destination pat
|
||||
|
||||
The adapter rejects symlink ancestors for reads and writes. Reads require regular files. Writes create parent directories and prefer atomic temp-file-plus-rename writes when requested. Walk output is sorted through the shared storage walker.
|
||||
|
||||
Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Forced replacement deletes the bounded destination bundle path.
|
||||
Managed cleanup and normal replacement delete only managed output paths plus `.distributor.json`. Merge publication retains omitted managed paths and may overwrite existing managed paths. Forced replacement deletes the bounded destination bundle path.
|
||||
|
||||
## Boundaries
|
||||
|
||||
|
||||
@@ -4,23 +4,23 @@ Audience: developers and LLM coding agents changing `internal/app`.
|
||||
|
||||
## Purpose
|
||||
|
||||
`internal/app` owns top-level application use cases: run, single-pipeline run, staged-source run, validate, inspect, manifest creation, and HTTP upload serving. It coordinates config loading, secret resolution, backend construction, source discovery, destination selection, publish planning/execution, notification handoff, output projection, and upload coordination.
|
||||
`internal/app` owns top-level application use cases: run, single-pipeline run, staged-source run, validate, inspect, manifest creation, reconcile-state planning/repair, prune planning/execution, and HTTP upload serving. It coordinates config loading, secret resolution, backend construction, source discovery, destination selection, publish planning/execution, state repair reporting, retention prune reporting, notification handoff, output projection, and upload coordination.
|
||||
|
||||
## Inputs And Outputs
|
||||
|
||||
Inputs include app option structs, contexts, config paths, pipeline ids, local source roots, dry-run/force flags, output format, stdout writers, HTTP requests, and optional notifier implementations.
|
||||
|
||||
Outputs include `RunReport`, validate/inspect/manifest results, CLI text/JSON projections, HTTP upload responses, upload status records, and errors. Destination-scoped failures can return a partial run report plus an aggregated error; fatal setup failures return before a complete report exists.
|
||||
Outputs include `RunReport`, `ReconcileStateReport`, `PrunePlanReport`, `PruneReport`, validate/inspect/manifest results, CLI text/JSON projections, HTTP upload responses, upload status records, and errors. Destination-scoped failures can return a partial run report plus an aggregated error; fatal setup failures return before a complete report exists.
|
||||
|
||||
## Boundaries
|
||||
|
||||
`internal/app` wires packages together but does not own manifest validation rules, destination state comparison, storage path rules, publish safety policy, transform rendering, config schema validation, or backend protocol behavior.
|
||||
|
||||
User-facing command parsing stays in `internal/cli`. User-facing config reference stays in `docs/config.md`. External contracts live under `docs/integrations/`.
|
||||
User-facing command parsing stays in `internal/cli`, including `reconcile-state` and `prune` flag validation and help text. User-facing config reference stays in `docs/config.md`. External contracts live under `docs/integrations/`.
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, and transfer policy.
|
||||
The package consumes the loaded `config.Config`: `server.http`, `secrets.directory`, pipeline ids, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, retention policy, and transfer policy.
|
||||
|
||||
Config fields are validated and defaulted by `internal/config` before app workflows use them.
|
||||
|
||||
@@ -34,13 +34,19 @@ The app layer registers default transforms, including Markdown-to-HTML, and supp
|
||||
|
||||
Run workflows discover and validate source bundles through `internal/bundle`. Destination state actions are prepared and written through `internal/publish` and `internal/state`; the app layer records report projections of those actions and results.
|
||||
|
||||
Reconcile-state workflows load one configured pipeline/destination selector, open that destination root, parse the root `.distributor.json`, and report missing managed output records plus unmanaged storage entries. Managed output existence checks use storage `Stat`; unmanaged reporting uses bounded storage `Walk` and excludes `.distributor.json` plus all paths already recorded as managed. Apply mode removes missing managed output records from state and rewrites valid state only; dry-run reports the same repair without writing. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope. It does not validate output digests, delete destination files, adopt unmanaged files, or rewrite invalid or mismatched state.
|
||||
|
||||
Prune planning consumes a parsed destination state document and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state.
|
||||
|
||||
Prune execution loads one configured pipeline/destination selector, opens that destination root, parses the root `.distributor.json`, and builds a plan from the destination retention policy. Dry-run returns the same planned and preserved managed output records without deleting files or rewriting state. Apply mode deletes only planned managed output paths, never unmanaged files or `.distributor.json`, then removes confirmed deleted records from state and updates the state timestamp. If a delete fails after earlier deletes succeeded, it rewrites state only for the confirmed deletions and preserves records for the failed and unattempted outputs so a retry remains accurate. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope.
|
||||
|
||||
HTTP uploads stage and validate archives before enqueueing a pipeline run with a local staged source root. Go producers can use the public `pkg/upload` package to create client-side gzip tar uploads for this server contract; `internal/app` remains the server-side orchestration boundary and does not import that producer package.
|
||||
|
||||
Upload idempotency is owned by the upload coordinator. Optional `Idempotency-Key` values are scoped to the authenticated pipeline. The coordinator reserves a key while staging is in progress, records the accepted run id with the validated source manifest identity after staging succeeds, returns the original accepted record for the same key and same manifest, and rejects the same key with a different manifest as a conflict.
|
||||
Upload idempotency is owned by the upload coordinator. Optional `Idempotency-Key` values are scoped to token id, pipeline id, and key. The coordinator reserves a key while staging is in progress, records the accepted run id with the validated source manifest identity after staging succeeds, returns the original accepted record for the same scoped key and same manifest, and rejects the same scoped key with a different manifest as a conflict.
|
||||
|
||||
## Skip And Resume Behavior
|
||||
|
||||
Fan-out destinations are independent. A destination failure is recorded and does not prevent later destinations from being attempted. Dry-run builds plans and reports without destination writes, destination state writes, notifier calls, or SSH known-host persistence.
|
||||
Fan-out destinations are independent. A destination failure is recorded and does not prevent later destinations from being attempted. Run dry-run builds plans and reports without destination writes, destination state writes, notifier calls, or SSH known-host persistence. Reconcile-state dry-run reports missing managed records and unmanaged entries without rewriting state. Prune dry-run reports planned managed output deletes without deleting outputs or rewriting state.
|
||||
|
||||
HTTP upload status is in memory. Accepted jobs move through accepted, queued, running, succeeded, or failed states and expire after configured retention. Upload idempotency records are also memory-only, expire with the completed status record for their accepted run, and are cleared by process restart.
|
||||
|
||||
@@ -48,11 +54,18 @@ HTTP upload status is in memory. Accepted jobs move through accepted, queued, ru
|
||||
|
||||
Runtime setup fails for config load, config validation, secret loading, or credential resolution errors. Source setup failures stop the affected run before destination planning. Destination open, planning, execution, and notification failures are recorded as destination failures where a partial result exists.
|
||||
|
||||
Reconcile-state setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root all-owner repair still uses the selected destination to identify the root, then applies repair across owners inside that root. Invalid, unreadable, or ambiguous state fails before any rewrite.
|
||||
|
||||
Prune setup fails unless the caller supplies a pipeline id and destination id that select one configured destination root. Single-owner state must match that pipeline/destination owner. Shared-root pruning is scoped to the selected owner and preserves unrelated owners. Invalid, unreadable, or ambiguous state fails before deletes or rewrites. Delete failures return a report with confirmed deletions and the failed output.
|
||||
|
||||
HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Upload requests can fail during authentication, idempotency-key validation, content-type validation, idempotency conflict checks, queue admission, archive staging, source validation, or later publish execution.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/app/*_test.go`
|
||||
- `internal/app/prune_test.go`
|
||||
- `internal/cli/reconcile_state_test.go`
|
||||
- `internal/cli/prune_test.go`
|
||||
- `internal/cli/root_test.go`
|
||||
- `internal/config/*_test.go`
|
||||
- `internal/ingest/*_test.go`
|
||||
@@ -67,3 +80,5 @@ HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Up
|
||||
- Upload admission stages and validates a bundle before returning a run id.
|
||||
- Idempotent upload retries compare normalized source manifest identity, not archive bytes.
|
||||
- Runtime backend registration remains app-owned.
|
||||
- Reconcile-state repairs state records only; it never deletes or adopts destination files.
|
||||
- Prune execution deletes managed output paths only and preserves failed records for retry.
|
||||
|
||||
@@ -18,7 +18,7 @@ The canonical user-facing config reference is `docs/config.md`.
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, and transfer policy.
|
||||
The package defines all user-visible config fields: `server.http`, `secrets`, `pipelines`, source and destination backend fields, validation policy, publish policy, transform policy, path mapping, links, state policy, reconciliation policy, retention policy, and transfer policy.
|
||||
|
||||
## Adapters Used
|
||||
|
||||
@@ -26,7 +26,7 @@ No external storage adapters are used directly. The package exposes normalized c
|
||||
|
||||
## State And Manifest Behavior
|
||||
|
||||
The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as publish/transform combinations, links, transfer policy, backend roots, S3 prefix shape, and HTTP upload source settings.
|
||||
The package does not parse source manifests or destination state. It validates config values that later affect manifest validation and destination state, such as publish/transform combinations, links, state policy, reconciliation policy, retention policy, transfer policy, backend roots, S3 prefix shape, and HTTP upload source settings.
|
||||
|
||||
## Skip And Resume Behavior
|
||||
|
||||
|
||||
@@ -8,19 +8,19 @@ Audience: developers and LLM coding agents changing `internal/publish`.
|
||||
|
||||
## Inputs And Outputs
|
||||
|
||||
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, transformer resolver, transfer policy, distributor version, and force flag.
|
||||
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, destination bundle path, path mapping mode, publish policy, transform policy, optional link policy, state policy, reconciliation policy, transformer resolver, transfer policy, distributor version, and force flag.
|
||||
|
||||
Output from planning is a `Plan` with action, reason, destination identity, selected outputs, optional existing state, optional primary URL, and force metadata. Execution writes selected source outputs, generated outputs, and `.distributor.json` for executable publish or replacement actions.
|
||||
Output from planning is a `Plan` with action, reason, destination identity, selected outputs, state mode, owner scope, reconciliation mode, optional existing single-owner or shared-root state, optional primary URL, and force metadata. Shared-root plans also expose other-owner outputs to preserve, current-owner outputs retained by merge, current-owner outputs deleted by replace, and current-owner outputs to write. Execution writes selected source outputs, generated outputs, and `.distributor.json` for executable publish or replacement actions.
|
||||
|
||||
## Boundaries
|
||||
|
||||
The package does not parse CLI flags, load config files, open concrete adapters, discover source bundles, select fixed-path bundle candidates, register transforms, or render command output. The app layer supplies validated request data and concrete dependencies.
|
||||
The package does not parse CLI flags, load config files, open concrete adapters, discover source bundles, select fixed-path bundle candidates, register transforms, prune retained outputs, or render command output. The app layer supplies validated request data and concrete dependencies.
|
||||
|
||||
External destination state semantics are documented in `docs/integrations/destination-state.md`.
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
The package consumes already-defaulted config values for destination `publish`, `transform`, `links`, `transfer`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
|
||||
The package consumes already-defaulted config values for destination `publish`, `transform`, `links`, `state`, `reconciliation`, `transfer`, and path mapping mode. It uses `config.ValidatePublishTransformPolicy` for publish/transform consistency.
|
||||
|
||||
## Adapters Used
|
||||
|
||||
@@ -30,19 +30,25 @@ The package depends on `internal/storage.Backend` for source and destination IO,
|
||||
|
||||
Planning inspects destination state through `internal/state`, compares it with the source manifest, and maps comparison outcomes plus transfer policy into actions: `publish_new`, `replace_older`, `force_replace`, `skip_same`, `skip_destination_newer`, `fail_conflict`, or `fail_unmanaged`.
|
||||
|
||||
Execution writes destination state after selected outputs are written. Destination state includes copied source output metadata, generated output metadata, embedded source manifest, link metadata when configured, pipeline id, destination id, and publication timestamp.
|
||||
Single-owner destinations compare the whole destination state against the configured pipeline and destination ids. Shared-root destinations compare only the current owner scope, keyed by pipeline id and destination id. An absent shared-root owner is publishable for that owner unless a planned output collides with unmanaged storage content. Planned writes to a path owned by another shared-root owner fail as conflicts.
|
||||
|
||||
Execution writes destination state after selected outputs are written. Destination state includes copied source output metadata, generated output metadata, output timestamps, embedded source manifest, reconciliation metadata, link metadata when configured, pipeline id, destination id, and publication timestamps.
|
||||
|
||||
## Skip And Resume Behavior
|
||||
|
||||
`skip_same` and `skip_destination_newer` execute as no-ops. Normal replacement removes only managed output paths from existing state plus `.distributor.json`; this allows retries without broad deletion. Failed writes trigger cleanup of outputs written during that failed attempt where practical.
|
||||
`skip_same` and `skip_destination_newer` execute as no-ops. Replacement-mode single-owner updates remove managed output paths from existing state plus `.distributor.json`, verify the destination is empty, and write state whose outputs are exactly the new plan. Replacement-mode shared-root updates remove only current-owner omitted outputs and preserve unrelated owners. Merge-mode updates retain omitted managed outputs, overwrite only paths already recorded as managed, reject unmanaged destination path collisions, and write cumulative output state. Failed writes trigger cleanup where practical; merge cleanup removes only newly created outputs from the failed attempt.
|
||||
|
||||
Shared-root execution writes schema version `3` state. It preserves unrelated owner records and outputs, updates only the publishing owner metadata, preserves root `created_at`, and updates root `updated_at` after successful state writes. Compatible single-owner state for the same pipeline and destination is converted to shared-root state on successful publish.
|
||||
|
||||
Forced replacement is explicit per request and deletes the bounded destination bundle path before writing new outputs and state.
|
||||
|
||||
Retention pruning is not part of publish execution and does not run automatically after a successful publish. The app-level prune workflow uses destination state after publication to select managed outputs for deletion.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Planning fails for incomplete requests, invalid publish/transform policy, output path collisions, invalid destination state, unmanaged destination content without force, conflict outcomes not allowed by transfer policy, unresolved transforms, invalid Markdown output selection, and invalid link URL planning.
|
||||
Planning fails for incomplete requests, invalid publish/transform policy, invalid state mode, invalid reconciliation mode, output path collisions, invalid destination state, unmanaged destination content without force, shared-root owner path conflicts, conflict outcomes not allowed by transfer policy, unresolved transforms, invalid Markdown output selection, and invalid link URL planning.
|
||||
|
||||
Execution fails on delete, read, transform output, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable publish or replacement actions.
|
||||
Execution fails on delete, read, transform output, unmanaged merge path collision, shared-root ownership conflict, write, state validation, state serialization, or context errors. Execution refuses actions that are not executable publish or replacement actions.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
@@ -56,7 +62,11 @@ Execution fails on delete, read, transform output, write, state validation, stat
|
||||
- Planning is deterministic for the same request and destination state.
|
||||
- Destination bundle paths are caller-supplied and backend-root-relative.
|
||||
- URL generation uses URL path semantics and never infers public URLs from backend config.
|
||||
- Normal replacement deletes only managed paths recorded in existing state plus `.distributor.json`.
|
||||
- Replacement reconciliation deletes only managed paths recorded in existing state plus `.distributor.json` for single-owner state, and only current-owner omitted outputs for shared-root state.
|
||||
- Merge reconciliation never adopts unmanaged content.
|
||||
- Merge state output records are cumulative for the single owner.
|
||||
- Shared-root planning is owner-scoped and preserves unrelated owner outputs.
|
||||
- Shared-root execution writes owner-scoped changes without deleting unrelated owners.
|
||||
- Forced replacement deletes only within the supplied destination bundle path.
|
||||
- Destination state is written after selected outputs are written.
|
||||
- Transform resolution stays behind a caller-supplied interface.
|
||||
|
||||
@@ -18,7 +18,7 @@ The external destination state contract is documented in `docs/integrations/dest
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
None directly. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
|
||||
`internal/state` uses config state mode and reconciliation mode constants for destination state validation and legacy state normalization. Destination ids, pipeline ids, and link URLs originate from config but are supplied as values by callers.
|
||||
|
||||
## Adapters Used
|
||||
|
||||
@@ -26,22 +26,44 @@ None.
|
||||
|
||||
## State And Manifest Behavior
|
||||
|
||||
`.distributor.json` schema version is `1`. Required fields are `pipeline_id`, `destination_id`, `published_at`, `source.manifest`, and `outputs`. `distributor_version` and `links` are optional.
|
||||
`.distributor.json` schema version is `2` for newly written single-owner state. Required fields are `pipeline_id`, `destination_id`, `published_at`, `created_at`, `updated_at`, `state.mode`, `reconciliation.mode`, `source.manifest`, and `outputs`. `distributor_version` and `links` are optional.
|
||||
|
||||
Embedded source manifests are parsed and validated through `internal/bundle`, which delegates source manifest semantics to `pkg/bundle`. Output records require clean paths, `source` or `generated` kind, valid source paths, lowercase SHA-256 digests, non-negative sizes, and transform ids for generated outputs. Stored URLs must pass `internal/link` validation.
|
||||
Schema version `1` state remains readable. Parsing infers `state.mode: single_owner`, `reconciliation.mode: replace`, top-level `created_at` and `updated_at` from `published_at`, and per-output timestamps from `published_at`.
|
||||
|
||||
Schema version `3` is shared-root state. It records `state.mode: shared_root`, shared state timestamps, owner records keyed by pipeline id and destination id, each owner's latest source manifest and reconciliation metadata, optional owner primary links, and output records for every managed path. Shared-root output records include owner ids and compact source identity fields for source id, digest, and creation time.
|
||||
|
||||
Shared-root publish conversion is explicit. Compatible single-owner state for the same pipeline and destination can be projected into the current owner scope by publish execution. Unrelated single-owner state remains a conflict.
|
||||
|
||||
Embedded source manifests are parsed and validated through `internal/bundle`, which delegates source manifest semantics to `pkg/bundle`. Output records require clean paths, `source` or `generated` kind, valid source paths, lowercase SHA-256 digests, non-negative sizes, created and updated timestamps, and transform ids for generated outputs. Stored URLs must pass `internal/link` validation.
|
||||
|
||||
The package also provides helpers for finding output records by path, projecting planned publish outputs into timestamped state outputs, merging retained and newly planned output records, computing managed output paths from single-owner state, removing missing managed output records from single-owner state, and building prune candidates from managed outputs.
|
||||
|
||||
For shared-root state, helpers parse either state shape, identify the current owner scope, return an owner's latest source manifest, list managed paths for one owner or all owners, detect path ownership conflicts, project planned owner outputs, merge one owner's planned outputs while preserving unrelated owners, replace one owner's outputs by removing that owner's omitted outputs, remove missing managed output records for either the current owner or every owner, and build owner-scoped prune candidates.
|
||||
|
||||
Shared-root helper projections preserve output `created_at` for existing managed paths and use the current publication time for rewritten `updated_at`. Root-level `created_at` preservation is owned by publish execution.
|
||||
|
||||
## Skip And Resume Behavior
|
||||
|
||||
Comparison is pure. It returns outcomes for absent state, unmanaged content, invalid state, pipeline/destination mismatch, same source manifest, older destination, newer destination, same-created digest conflict, and different source id conflict. It does not decide whether to skip, replace, force, or fail; publish planning maps outcomes to actions.
|
||||
|
||||
Shared-root comparison is owner-scoped. It compares only the owner keyed by the current pipeline id and destination id, treats an absent owner as absent destination state for that owner, and can compare compatible single-owner state for the current owner without converting unrelated single-owner state.
|
||||
|
||||
Missing-output removal helpers are pure state transformations used by app-level state repair. They remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers.
|
||||
|
||||
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution and the `prune` command use the missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid embedded manifests, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transforms, invalid URLs, invalid digests, and negative sizes.
|
||||
Parsing rejects invalid JSON, trailing data, missing required fields, invalid timestamps, invalid state mode, invalid reconciliation mode, invalid embedded manifests, duplicate owners, duplicate outputs, invalid output paths, unsupported output kinds, missing generated transforms, invalid URLs, invalid digests, negative sizes, and shared-root outputs whose owner is not registered.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/state/distributor_test.go`
|
||||
- `internal/state/shared_root_test.go`
|
||||
- `internal/state/prune_test.go`
|
||||
- `internal/state/compare_test.go`
|
||||
- `internal/app/reconcile_state_test.go`
|
||||
- `internal/cli/reconcile_state_test.go`
|
||||
- `internal/publish/*_test.go`
|
||||
|
||||
## Architectural Invariants
|
||||
@@ -49,6 +71,13 @@ Parsing rejects invalid JSON, trailing data, missing required fields, invalid ti
|
||||
- `.distributor.json` is the destination sentinel and state record.
|
||||
- Comparison does not mutate storage.
|
||||
- Embedded source manifests use the source bundle contract.
|
||||
- Newly written single-owner state uses schema version `2`.
|
||||
- Schema version `1` state remains readable as replacement-mode single-owner state.
|
||||
- Schema version `3` shared-root state is parsed and validated without converting unrelated single-owner state.
|
||||
- Shared-root owner updates preserve unrelated owners and reject planned path collisions with other owners.
|
||||
- Missing-output repair helpers preserve unrelated owner records and outputs.
|
||||
- Prune planning uses output `updated_at` and preserves unrelated shared-root owners.
|
||||
- Generated outputs always record a transform id.
|
||||
- Output records always carry created and updated timestamps after parsing.
|
||||
- Stored URLs are optional and must be absolute HTTP or HTTPS URLs when present.
|
||||
- `distributor_version` is diagnostic metadata, not a comparison key.
|
||||
|
||||
@@ -26,13 +26,15 @@ Local, SSH/SFTP, and S3-compatible adapters implement `storage.Backend`. `intern
|
||||
|
||||
## State And Manifest Behavior
|
||||
|
||||
Storage owns `.distributor.json` path helpers through `StateFileName`, `StatePath`, and `ManagedBundleTargets`. It does not parse source manifests or destination state.
|
||||
Storage owns `.distributor.json` path helpers through `StateFileName`, `StatePath`, `ManagedOutputTargets`, and `ManagedBundleTargets`. It does not parse source manifests or destination state.
|
||||
|
||||
Logical paths are slash-separated and relative to a backend root. Prefix validation allows an empty prefix to mean the backend root; file path validation requires a non-empty path.
|
||||
|
||||
Reconcile-state callers use `Stat` to check whether managed output paths still exist and bounded recursive `Walk` to report unmanaged entries under a selected destination root. Prune execution callers use `DeleteManagedOutputs` for selected managed output records so `.distributor.json` is not part of the deletion target set. Storage does not decide whether entries are managed; callers compare entries against destination state.
|
||||
|
||||
## Skip And Resume Behavior
|
||||
|
||||
Storage has no publication skip policy. It supplies `HasAny` for unmanaged-content checks, `DeleteManagedBundle` target construction for normal replacement cleanup, and `DeletePrefix` semantics for explicit forced replacement.
|
||||
Storage has no publication skip policy. It supplies `HasAny` for unmanaged-content checks, `Stat` and `Walk` for state repair inspection, `DeleteManagedOutputs` target construction for managed output cleanup, `DeleteManagedBundle` target construction for normal replacement cleanup, and `DeletePrefix` semantics for explicit forced replacement.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
@@ -42,6 +44,7 @@ Storage errors use typed categories: not found, already exists, not empty, inval
|
||||
|
||||
- `internal/storage/*_test.go`
|
||||
- `internal/storage/fake/*_test.go`
|
||||
- `internal/app/reconcile_state_test.go`
|
||||
- `internal/adapters/local/*_test.go`
|
||||
- `internal/adapters/ssh/*_test.go`
|
||||
- `internal/adapters/s3/*_test.go`
|
||||
@@ -51,6 +54,6 @@ Storage errors use typed categories: not found, already exists, not empty, inval
|
||||
- Logical paths are clean relative slash-separated paths confined to the backend root.
|
||||
- Core packages never import concrete adapters.
|
||||
- `storage.List` returns deterministic sorted entries.
|
||||
- Managed deletion targets are recorded outputs plus `.distributor.json`.
|
||||
- Managed output deletion targets are recorded outputs only; managed bundle deletion targets are recorded outputs plus `.distributor.json`.
|
||||
- Prefix deletion is bounded to the requested logical prefix.
|
||||
- Runtime registration remains app-owned.
|
||||
|
||||
@@ -56,7 +56,7 @@ Destination path mapping controls where each source bundle is published beneath
|
||||
|
||||
Fixed destinations select the newest discovered source bundle by manifest `created` timestamp. If multiple bundles have the same timestamp, the source-root-relative bundle path in ascending order wins.
|
||||
|
||||
Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination state record. It records the pipeline id, destination id, publication time, source manifest, copied outputs, generated outputs, and optional public URL metadata.
|
||||
Published destination bundle paths contain `.distributor.json`. See [Destination State Contract](integrations/destination-state.md). This file is both the managed sentinel and the destination state record. It records the pipeline id, destination id, publication time, state mode, reconciliation mode, source manifest, copied outputs, generated outputs, output timestamps, and optional public URL metadata.
|
||||
|
||||
`manifest.json` from the source bundle is not copied as destination state.
|
||||
|
||||
@@ -71,19 +71,90 @@ Published destination bundle paths contain `.distributor.json`. See [Destination
|
||||
- Invalid destination state, identity mismatch, different source id, or same-created digest mismatch: fail by default.
|
||||
- Content without `.distributor.json`: fail as unmanaged content by default.
|
||||
|
||||
Normal replacement deletes only managed output paths recorded in `.distributor.json` plus the state file, then verifies the destination bundle path is empty before writing new outputs and state.
|
||||
When destination state is older than the source, `transfer.on_destination_older` controls whether publication may proceed and `reconciliation.mode` controls how managed outputs are updated.
|
||||
|
||||
If a write fails after some outputs were written, `distributor` attempts to delete outputs from that failed attempt so a retry does not treat partial outputs as unmanaged content. Operators should still inspect the destination after a failed write before retrying.
|
||||
`reconciliation.mode: replace` is the default. It deletes only managed output paths recorded in `.distributor.json` plus the state file, verifies the destination bundle path is empty, then writes the newly planned outputs and state. The new state `outputs` array is exactly the newly planned output set.
|
||||
|
||||
`reconciliation.mode: merge` retains prior managed outputs that are omitted from the new plan. It overwrites planned paths only when those paths are already recorded in existing state as managed. If a newly planned path already exists in storage but is not recorded in state, publication fails as an unmanaged path collision. The new state `outputs` array is the cumulative managed output set.
|
||||
|
||||
For both modes, retained or overwritten paths are identified only from `.distributor.json`; unmanaged files are not adopted.
|
||||
|
||||
For `state.mode: shared_root`, one destination root may contain outputs from multiple pipeline/destination owners. Comparisons, replacement, and merge retention are scoped to the current owner. Outputs owned by other owners are preserved. A planned output path owned by another owner fails as a conflict, and a planned path that exists in storage but is not recorded in state fails as unmanaged content by default.
|
||||
|
||||
If `state.mode: shared_root` is configured on a destination whose existing single-owner state belongs to the same pipeline and destination, the next successful publish converts that state file to shared-root schema. Existing single-owner state for a different pipeline or destination remains a conflict.
|
||||
|
||||
If a write fails after some outputs were written, `distributor` attempts cleanup before returning the error. In `replace` mode, cleanup removes outputs written during that failed attempt. In `merge` mode, cleanup removes only newly created outputs from that failed attempt; overwritten managed outputs are left in place because they previously belonged to the managed set. Operators should still inspect the destination after a failed write before retrying.
|
||||
|
||||
Fan-out destinations are independent. If one destination fails after planning or execution begins, later destinations are still attempted. The command exits non-zero if any destination failed.
|
||||
|
||||
## Destination State Repair
|
||||
|
||||
Use `reconcile-state` when `.distributor.json` still records managed outputs that no longer exist in destination storage. This repairs the state record only; it does not restore missing files.
|
||||
|
||||
Preview the repair first:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor reconcile-state \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id> \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
Apply after reviewing the report:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor reconcile-state \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id>
|
||||
```
|
||||
|
||||
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, checks each managed output path with storage metadata, reports missing managed outputs, and reports unmanaged entries under that root. It excludes `.distributor.json` and already managed paths from unmanaged reporting.
|
||||
|
||||
Without `--dry-run`, it removes missing managed output records from valid state and rewrites `.distributor.json`. It never deletes destination files, adopts unmanaged files, validates output digests, or rewrites invalid or mismatched state.
|
||||
|
||||
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, repair is scoped to the selected owner by default. Add `--all-owners` only when every owner in the selected shared-root state should have missing managed output records removed.
|
||||
|
||||
## Managed Output Pruning
|
||||
|
||||
Use `prune` when a destination config has `retention.prune.enabled: true` and old managed outputs should be removed according to that configured policy. Pruning is never automatic after publish.
|
||||
|
||||
Preview selected managed outputs first:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id> \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
Apply after reviewing the report:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id> \
|
||||
--apply
|
||||
```
|
||||
|
||||
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, and plans from the selected destination's `retention.prune` policy. It uses managed output `updated_at` timestamps. When both `keep_latest` and `older_than` are configured, it preserves the newest `keep_latest` outputs before applying the age policy.
|
||||
|
||||
`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate.
|
||||
|
||||
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
|
||||
|
||||
## Dry Runs And Output Review
|
||||
|
||||
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, or SSH `known_hosts` entries.
|
||||
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, or SSH `known_hosts` entries. For reconciliation, dry runs report the same high-level action labels as execution; inspect the configured destination's `reconciliation.mode` to determine whether `replace_older` will replace the managed set or merge into it.
|
||||
|
||||
For shared-root destinations, dry runs are owner-scoped. A `replace_older` action replaces or merges only the current owner according to `reconciliation.mode`; unrelated owners remain managed by the shared-root state.
|
||||
|
||||
Review these action labels before publishing:
|
||||
|
||||
- `publish_new`: destination is empty and unmanaged.
|
||||
- `publish_new`: destination state is absent, or a shared-root owner is absent and planned paths are publishable.
|
||||
- `replace_older`: destination state is older than the source.
|
||||
- `skip_same`: destination state already matches the source.
|
||||
- `skip_destination_newer`: destination state is newer than the source and is skipped.
|
||||
@@ -110,11 +181,13 @@ Forced replacement can claim unmanaged non-empty destination paths. State confli
|
||||
|
||||
Forced replacement deletes the current destination bundle path before writing outputs and state. It does not delete parent paths, sibling paths, or storage outside the destination bundle path. For fixed destinations, the destination bundle path is the backend root, so a forced replacement can clear that configured root.
|
||||
|
||||
For shared-root destinations, forced replacement also deletes the configured destination bundle path before writing new shared-root state. This removes unrelated owners inside that destination root. Preview with `--dry-run --force` and confirm the destination path before applying.
|
||||
|
||||
`--force` applies only to the current invocation. There is no config field that enables forced replacement by default.
|
||||
|
||||
## HTTP Upload Operation
|
||||
|
||||
The [HTTP Upload API Contract](integrations/http-upload.md) defines request and response details. `distributor serve` runs the HTTP upload API for pipelines whose source backend is `http_upload`. Each bearer token maps to exactly one configured upload pipeline. Token values come from the process environment or `secrets.directory`, not from YAML literal values.
|
||||
The [HTTP Upload API Contract](integrations/http-upload.md) defines request and response details. `distributor serve` runs the HTTP upload API for pipelines whose source backend is `http_upload`. Top-level `upload_tokens` authenticate producers and allow one or more upload pipelines. Token values come from the process environment or `secrets.directory`, not from YAML literal values.
|
||||
|
||||
Start the maintained local example:
|
||||
|
||||
@@ -132,36 +205,25 @@ curl http://127.0.0.1:8080/healthz
|
||||
Upload one tar or tar.gz source bundle archive:
|
||||
|
||||
```sh
|
||||
curl -X POST http://127.0.0.1:8080/upload \
|
||||
curl -X POST http://127.0.0.1:8080/v1/pipelines/example-http-upload/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/gzip" \
|
||||
--data-binary @bundle.tar.gz
|
||||
```
|
||||
|
||||
For safe producer retries, include an idempotency key that is stable for the producer operation:
|
||||
For safe producer retries, include an idempotency key that is stable for the same producer run and different for each distinct run:
|
||||
|
||||
```sh
|
||||
curl -X POST http://127.0.0.1:8080/upload \
|
||||
curl -X POST http://127.0.0.1:8080/v1/pipelines/example-http-upload/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/gzip" \
|
||||
-H "Idempotency-Key: producer.run.20260604T120000Z" \
|
||||
--data-binary @bundle.tar.gz
|
||||
```
|
||||
|
||||
Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. The package sends `Idempotency-Key` on every upload, derives `/upload` from the configured endpoint, and reuses the same key and replayable request body for safe retries:
|
||||
Go producer applications can use `pkg/upload` instead of constructing archives and HTTP requests directly. See [Upstream Producer Integration](consumers/api.md) for the copyable producer implementation guide.
|
||||
|
||||
```go
|
||||
client, err := upload.NewClient(upload.ClientOptions{
|
||||
Endpoint: "http://127.0.0.1:8080",
|
||||
Token: token,
|
||||
})
|
||||
result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
|
||||
Root: "examples/source-bundle",
|
||||
IdempotencyKey: "producer.run.20260604T120000Z",
|
||||
})
|
||||
```
|
||||
|
||||
The maintained example client uses the local upload server and reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`. It generates an idempotency key by default; set `DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY` when a retry must be stable across separate process runs.
|
||||
The maintained example client uses the local upload server, reads the token from `DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN`, and defaults the pipeline id to `example-http-upload`. Set `DISTRIBUTOR_EXAMPLE_UPLOAD_PIPELINE_ID` or pass a second argument to use another configured upload pipeline. It generates an idempotency key by default; set `DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY` when retrying the same producer run across separate process runs.
|
||||
|
||||
```sh
|
||||
go run ./examples/upload-client
|
||||
@@ -183,7 +245,7 @@ Status values are `accepted`, `queued`, `running`, `succeeded`, and `failed`. Co
|
||||
|
||||
Upload admission is bounded by `server.http.queue_size`. Publication concurrency is bounded by `server.http.max_concurrency`, and the coordinator does not run two uploads for the same pipeline at the same time.
|
||||
|
||||
`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the authenticated pipeline. Reusing the same key with the same normalized source manifest returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`. If another request with the same key is still being staged before its manifest is known, the server returns a retryable `409 Conflict`. Idempotency records are memory-only and expire with completed upload status records.
|
||||
`Idempotency-Key` is optional for raw HTTP clients. When present, it is scoped to the token id, pipeline id, and key. Reusing the same key with the same normalized source manifest in that scope returns the original accepted run response and does not enqueue another run. Reusing the key with a different source manifest returns `409 Conflict`. If another request with the same key is still being staged before its manifest is known, the server returns a retryable `409 Conflict`. Idempotency records are memory-only and expire with completed upload status records.
|
||||
|
||||
The upload server accepts `application/x-tar`, `application/gzip`, and `application/x-gzip`. Archives are extracted into a temporary staging directory, must contain exactly one root-level `manifest.json`, and must validate as one complete source bundle before a run id is issued. Per-source `max_upload_size` bounds both uploaded archive size and extracted bundle size. The implementation also caps extracted file count.
|
||||
|
||||
@@ -205,7 +267,7 @@ S3 execution uses the AWS SDK for Go v2. See [S3-Compatible Storage Integration]
|
||||
|
||||
When explicit S3 credential variable names are configured, both must resolve to non-empty values through the process environment or `secrets.directory`. When omitted, the AWS SDK default credential chain is used as-is.
|
||||
|
||||
Normal replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Forced replacement deletes objects under the bounded destination bundle prefix. Distributor does not manage bucket versioning or delete markers.
|
||||
Normal single-owner replacement and failed-write cleanup delete only managed output objects recorded in `.distributor.json` plus the state object. Shared-root replacement deletes only current-owner omitted output objects and rewrites the shared state object. Merge publication retains omitted managed objects and may overwrite existing managed objects. Forced replacement deletes objects under the bounded destination bundle prefix. Distributor does not manage bucket versioning or delete markers.
|
||||
|
||||
## Secrets Operation
|
||||
|
||||
@@ -220,7 +282,10 @@ Use these recovery boundaries:
|
||||
- For source validation failures, regenerate the source bundle and manifest together.
|
||||
- For an empty or missing destination, rerun after fixing config or storage access.
|
||||
- For unmanaged destination content, move unrelated files aside or use a different destination path before publishing.
|
||||
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`.
|
||||
- For shared-root ownership conflicts, change one owner so it writes a different destination path, or use a separate destination root.
|
||||
- For missing managed output files recorded in state, run `reconcile-state --dry-run`, then apply `reconcile-state` if the missing files should no longer be considered managed.
|
||||
- For configured retention cleanup, run `prune --dry-run`, then apply `prune --apply` after reviewing the managed output list.
|
||||
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`. In merge mode, retained outputs may be intentional managed outputs from the prior state.
|
||||
- For state conflicts, verify the source, pipeline, destination, and existing `.distributor.json` before considering `--force`.
|
||||
- For HTTP upload failures, inspect `/runs/<run-id>` while retained; after expiry or restart, rely on destination state and logs/output from the publishing run.
|
||||
|
||||
|
||||
@@ -94,12 +94,19 @@ The source manifest should remain minimal. Routing, destination selection, publi
|
||||
|
||||
`manifest.json` from the source bundle is not copied to destinations as destination state.
|
||||
|
||||
Each destination bundle path is managed by `.distributor.json`. This file is both the destination sentinel and the destination state record. It records:
|
||||
Each destination bundle path is managed by `.distributor.json`. This file is both the destination sentinel and the destination state record.
|
||||
|
||||
Publish execution writes single-owner or shared-root destination state according to destination `state.mode`. In shared-root state, one `.distributor.json` records multiple pipeline/destination owners and every managed output carries its owner identity.
|
||||
|
||||
Single-owner state records:
|
||||
|
||||
- `distributor` state schema version;
|
||||
- pipeline id;
|
||||
- destination id;
|
||||
- publication timestamp;
|
||||
- state creation and update timestamps;
|
||||
- single-owner state mode;
|
||||
- reconciliation mode;
|
||||
- the normalized source manifest used for publication;
|
||||
- metadata for copied source outputs;
|
||||
- metadata for generated outputs, such as HTML files;
|
||||
@@ -110,11 +117,19 @@ A representative destination state file is:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"schema_version": 2,
|
||||
"distributor_version": "0.1.0",
|
||||
"pipeline_id": "weather-daily",
|
||||
"destination_id": "static-html",
|
||||
"published_at": "2026-05-30T11:12:00Z",
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z",
|
||||
"state": {
|
||||
"mode": "single_owner"
|
||||
},
|
||||
"reconciliation": {
|
||||
"mode": "replace"
|
||||
},
|
||||
"source": {
|
||||
"manifest": {
|
||||
"schema_version": 1,
|
||||
@@ -141,7 +156,9 @@ A representative destination state file is:
|
||||
"transform": "markdown_to_html",
|
||||
"sha256": "sha256:...",
|
||||
"size": 23456,
|
||||
"url": "https://reports.example.com/weather-daily/"
|
||||
"url": "https://reports.example.com/weather-daily/",
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -156,6 +173,8 @@ Destination comparison rules are based on `.distributor.json`:
|
||||
- Existing state has the same source id and same `created` but different digest: fail as a conflict.
|
||||
- Existing state has a different source id: fail as a conflict.
|
||||
|
||||
For older destination state, destination `reconciliation.mode` controls output updates. `replace` rewrites the managed output set to match the new plan. `merge` retains omitted managed outputs, overwrites only existing managed paths, and rejects planned paths that collide with unmanaged storage content.
|
||||
|
||||
## Publication and Transform Policy
|
||||
|
||||
Source files are canonical bundle artifacts. Transform outputs are derived publication artifacts.
|
||||
@@ -197,6 +216,7 @@ Use this current layout unless the project has a documented reason to differ:
|
||||
|
||||
- `cmd/distributor`: application entrypoint only.
|
||||
- `pkg/bundle`: public producer-facing source manifest model, digest logic, parsing, manifest building, complete local bundle writing, and local validation helpers.
|
||||
- `pkg/upload`: public producer-facing HTTP upload client built on `pkg/bundle`.
|
||||
- `internal/app`: application orchestration and top-level use cases.
|
||||
- `internal/cli`: CLI command definitions, flags, argument parsing, and command wiring.
|
||||
- `internal/config`: configuration structs, defaults, loading, precedence, and validation.
|
||||
@@ -323,9 +343,9 @@ Important tests include:
|
||||
|
||||
Documentation should follow the project documentation policy. Keep user docs focused on implemented behavior. Put future, planned, or aspirational work only under `docs/roadmap/`.
|
||||
|
||||
When changing architecture, config, CLI behavior, adapters, manifest/state contracts, transform behavior, publish behavior, or component contracts, update the relevant docs and examples in the same change.
|
||||
When changing architecture, config, CLI behavior, adapters, manifest/state contracts, transform behavior, publish behavior, public package/API behavior, or component contracts, update the relevant docs and examples in the same change.
|
||||
|
||||
The source manifest and destination `.distributor.json` schemas should have canonical documentation once implemented. Example configs should be valid and load-tested where practical.
|
||||
The source manifest and destination `.distributor.json` schemas should have canonical documentation once implemented. Producer-facing package and API workflows belong under `docs/consumers/`. Example configs should be valid and load-tested where practical.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`.
|
||||
- `internal/transform/markdown`: Markdown-to-HTML transform.
|
||||
- `internal/notify`: notification interface and current no-op notifier.
|
||||
- `internal/testutil`: shared test fixtures. Production code must not import this package.
|
||||
- `docs`: current user, operator, policy, internal, and roadmap documentation.
|
||||
- `docs`: current user, operator, consumer, integration, policy, internal, and roadmap documentation.
|
||||
- `examples`: copyable example configs and source bundles.
|
||||
|
||||
Do not create new top-level package families such as public `pkg/...` packages
|
||||
@@ -211,5 +211,7 @@ Follow `docs/policy/documentation.md`.
|
||||
- Keep `docs/config.md` canonical for user-facing config reference.
|
||||
- Keep `docs/cli.md` canonical for command syntax and workflows.
|
||||
- Keep `docs/operations.md` canonical for operational and recovery behavior.
|
||||
- Keep `docs/consumers/` canonical for public package and consumer API workflows.
|
||||
- Keep `docs/integrations/` canonical for external file-format and wire-protocol contracts.
|
||||
- Keep `docs/internal/` focused on implemented package contracts.
|
||||
- Update docs in the same change as behavior when public behavior, config, CLI, examples, or internal contracts change.
|
||||
- Update docs in the same change as behavior when public behavior, public packages/APIs, config, CLI, examples, or internal contracts change.
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
Project documentation must help five audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||
5. developers and LLM coding agents integrating this project from another codebase.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
|
||||
@@ -46,7 +47,9 @@ Canonical homes:
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- public API/package consumer guidance: `docs/consumers/`
|
||||
- implemented internals: `docs/internal/`
|
||||
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
@@ -119,6 +122,15 @@ Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Project with public packages or consumer APIs
|
||||
|
||||
Required:
|
||||
- `docs/consumers/api.md`
|
||||
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||
|
||||
Recommended:
|
||||
- copyable consumer examples under `examples/`, if practical
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
@@ -244,6 +256,33 @@ Each entry should include:
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/consumers/
|
||||
|
||||
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||
|
||||
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||
|
||||
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||
|
||||
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||
|
||||
1. intended consumer audience and use cases;
|
||||
2. required inputs supplied by operators or deployment configuration;
|
||||
3. recommended public package or API workflow;
|
||||
4. minimal copyable example;
|
||||
5. consumer responsibilities and boundaries;
|
||||
6. retry, idempotency, or status behavior, if applicable;
|
||||
7. links to package-specific docs and canonical integration contracts.
|
||||
|
||||
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||
|
||||
1. import path;
|
||||
2. intended use cases;
|
||||
3. primary types and functions needed by consumers;
|
||||
4. minimal examples;
|
||||
5. validation, error, retry, and boundary behavior;
|
||||
6. links to canonical file-format or wire-protocol contracts.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
@@ -289,7 +328,7 @@ Roadmap docs should not be confused with current behavior.
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
@@ -348,6 +387,7 @@ Before merging documentation changes, verify:
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Consumer-facing docs explain public APIs without duplicating integration contracts.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
|
||||
@@ -33,6 +33,21 @@ internal docs.
|
||||
- Resumable upload support.
|
||||
- Streaming upload protocols.
|
||||
|
||||
## Destination State Repair
|
||||
|
||||
- Unmanaged-file adoption workflow for destination state repair.
|
||||
- Digest-audit mode for managed destination outputs.
|
||||
- Explicit invalid-state repair workflow.
|
||||
- Whole-config state repair command.
|
||||
|
||||
## Destination Retention And Pruning
|
||||
|
||||
- One-off retention overrides for `prune`.
|
||||
- Automatic post-publish pruning.
|
||||
- Path/date parsing retention policies.
|
||||
- Group-level pruning by source publication.
|
||||
- Removal of empty state files.
|
||||
|
||||
## Destination Backends
|
||||
|
||||
- GitHub Gist destination backend support.
|
||||
@@ -68,6 +83,7 @@ internal docs.
|
||||
- Mutual TLS or other in-app identity mechanisms.
|
||||
- In-app TLS.
|
||||
- Public exposure defaults.
|
||||
- In-app upload rate limiting.
|
||||
- Browser UI.
|
||||
|
||||
## Boundaries
|
||||
@@ -76,5 +92,12 @@ internal docs.
|
||||
contract.
|
||||
- Current upload status, queue, and idempotency state are memory-only.
|
||||
- Producers submit complete tar or gzip-compressed tar source bundles today.
|
||||
- Producers do not choose destination ids, destination paths, transforms, links,
|
||||
publish policy, transfer policy, storage backends, reconciliation mode, state
|
||||
mode, or retention policy through upload requests.
|
||||
- Source manifests remain free of routing, destination, transform, credential,
|
||||
reconciliation, state, and retention data.
|
||||
- Public access policy, TLS termination, and rate limiting belong outside
|
||||
`distributor` unless a future implementation changes that boundary.
|
||||
- `distributor` is not a broad storage synchronization tool unless a future
|
||||
implementation explicitly changes that non-goal.
|
||||
|
||||
@@ -69,6 +69,42 @@ Safe fix: use either `distributor validate <path>` / `distributor inspect <path>
|
||||
|
||||
Reference: [CLI](cli.md#validate).
|
||||
|
||||
## Reconcile-State Selector Is Missing Or Wrong
|
||||
|
||||
Symptom: `reconcile-state requires --config`, `requires --pipeline`, `requires --destination`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
|
||||
|
||||
Likely cause: the command did not identify one configured destination root, or the selected root contains state for a different single-owner pipeline/destination.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor reconcile-state --help
|
||||
rg -n 'pipelines:|destinations:|id:' <config-path>
|
||||
cat <destination-path>/.distributor.json
|
||||
```
|
||||
|
||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. For unrelated single-owner state, use the correct config selector or a separate destination root; `reconcile-state` will not take over mismatched state.
|
||||
|
||||
Reference: [CLI](cli.md#reconcile-state).
|
||||
|
||||
## Prune Selector Or Mode Is Missing Or Wrong
|
||||
|
||||
Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
|
||||
|
||||
Likely cause: the command did not identify one configured destination root, did not choose exactly one execution mode, or the selected root contains state for a different single-owner pipeline/destination.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune --help
|
||||
rg -n 'retention:|prune:|pipelines:|destinations:|id:' <config-path>
|
||||
cat <destination-path>/.distributor.json
|
||||
```
|
||||
|
||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both. For unrelated single-owner state, use the correct config selector or a separate destination root; `prune` will not take over mismatched state.
|
||||
|
||||
Reference: [CLI](cli.md#prune).
|
||||
|
||||
## Output Format Is Invalid
|
||||
|
||||
Symptom: `format must be text or json`.
|
||||
@@ -182,6 +218,48 @@ Safe fix: verify the source and destination are intended to match. Use a separat
|
||||
|
||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
||||
|
||||
## Destination State References Missing Managed Outputs
|
||||
|
||||
Symptom: `reconcile-state --dry-run` reports `status=would_change` or JSON `missing_managed_outputs` entries.
|
||||
|
||||
Likely cause: files that were recorded as managed in `.distributor.json` were removed outside `distributor`, or a previous external cleanup removed destination files without updating state.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor reconcile-state \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id> \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
Safe fix: if the missing files should no longer be managed, rerun the same command without `--dry-run` to remove only the missing managed output records from `.distributor.json`. The command does not delete destination files or adopt unmanaged entries. If the files should exist, restore them from backup or republish instead.
|
||||
|
||||
Reference: [Operations](operations.md#destination-state-repair).
|
||||
|
||||
## Prune Reports No Planned Deletes
|
||||
|
||||
Symptom: `prune --dry-run` reports `planned=0` or JSON `planned_outputs: []`.
|
||||
|
||||
Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected shared-root owner has no eligible outputs.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
rg -n 'retention:|prune:|older_than:|keep_latest:' <config-path>
|
||||
go run ./cmd/distributor prune \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id> \
|
||||
--dry-run \
|
||||
--format json
|
||||
```
|
||||
|
||||
Safe fix: verify the selected destination's retention policy and the managed output `updated_at` timestamps in `.distributor.json`. Adjust config if the policy is too conservative, then rerun `--dry-run` before using `--apply`.
|
||||
|
||||
Reference: [Operations](operations.md#managed-output-pruning).
|
||||
|
||||
## Destination Is Newer Than Source
|
||||
|
||||
Symptom: `skip_destination_newer` or `destination is newer and replacement requires --force`.
|
||||
@@ -216,9 +294,9 @@ Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||
|
||||
## Output Path Collision
|
||||
|
||||
Symptom: `destination output path collision`.
|
||||
Symptom: `destination output path collision` or `merge output path ... exists but is not managed by destination state`.
|
||||
|
||||
Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`.
|
||||
Likely cause: publication would write two outputs to the same destination path, such as copying `report.html` while also generating `report.html` from `report.md`. For merge reconciliation, it can also mean a planned output path already exists in storage but is not recorded in `.distributor.json` as managed.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -226,10 +304,44 @@ Diagnostic:
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide.
|
||||
Safe fix: adjust source files or publish/transform policy so copied and generated outputs do not collide. For merge reconciliation, move unmanaged content aside, choose another destination path, or use replacement/forced replacement only when deleting the destination bundle path is intended.
|
||||
|
||||
Reference: [Configuration](config.md#publish-and-transform-policy).
|
||||
|
||||
## Shared-Root Ownership Conflict
|
||||
|
||||
Symptom: `fail_conflict` with a reason like `destination output path ... is owned by <pipeline>/<destination>`.
|
||||
|
||||
Likely cause: a `state.mode: shared_root` destination planned an output path already recorded in `.distributor.json` for another pipeline/destination owner.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
cat <destination-path>/.distributor.json
|
||||
go run ./cmd/distributor run --config <config-path> --dry-run
|
||||
```
|
||||
|
||||
Safe fix: change one owner so it writes a different output path, use a separate destination root, or intentionally replace the whole destination root only after previewing with `--dry-run --force`.
|
||||
|
||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
||||
|
||||
## Shared-Root Unmanaged Path Collision
|
||||
|
||||
Symptom: `fail_unmanaged` with a reason like `destination output path ... exists but is not managed by destination state`.
|
||||
|
||||
Likely cause: a `state.mode: shared_root` destination planned a new output path that already exists in storage but is not recorded as managed in `.distributor.json`.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
find <destination-path> -maxdepth 2 -print
|
||||
cat <destination-path>/.distributor.json
|
||||
```
|
||||
|
||||
Safe fix: move the unmanaged file aside, change the planned output path, or use forced replacement only when deleting the configured destination root is intended.
|
||||
|
||||
Reference: [Operations](operations.md#forced-replacement-workflow).
|
||||
|
||||
## Run Failed After Writing Some Files
|
||||
|
||||
Symptom: a destination write failed and the command exited non-zero after partial work.
|
||||
@@ -242,7 +354,7 @@ Diagnostic:
|
||||
find <destination-path> -maxdepth 2 -print
|
||||
```
|
||||
|
||||
Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. Rerun `--dry-run` before publishing again.
|
||||
Safe fix: inspect the destination bundle path printed in the error. `distributor` attempts to remove outputs from the failed attempt, but operators should verify the destination before retrying. In merge mode, previously managed retained or overwritten outputs may remain intentionally. Rerun `--dry-run` before publishing again.
|
||||
|
||||
Reference: [Operations](operations.md#destination-state-and-retry-behavior).
|
||||
|
||||
@@ -400,7 +512,7 @@ Reference: [Configuration](config.md#serverhttp).
|
||||
|
||||
Symptom: `upload token environment variable ... is not set`, `... is empty`, or `upload token environment variables ... resolve to the same value`.
|
||||
|
||||
Likely cause: an `http_upload` source references a missing/empty `token_env`, or two upload pipelines resolve to the same bearer token.
|
||||
Likely cause: a top-level upload token record references a missing or empty `token_env`, or two token records resolve to the same bearer token.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -410,20 +522,20 @@ env | cut -d= -f1 | rg '^<token-variable>$'
|
||||
ls -l <secrets-directory>/<token-variable>
|
||||
```
|
||||
|
||||
Safe fix: provide one distinct non-empty token value per upload pipeline through the process environment or `secrets.directory`. Do not put literal tokens in YAML.
|
||||
Safe fix: provide one distinct non-empty token value per upload token record through the process environment or `secrets.directory`. Do not put literal tokens in YAML.
|
||||
|
||||
Reference: [Configuration](config.md#http-upload-source-backend).
|
||||
Reference: [Configuration](config.md#upload_tokens).
|
||||
|
||||
## Upload Request Is Unauthorized
|
||||
|
||||
Symptom: `POST /upload` returns `401`.
|
||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `401`.
|
||||
|
||||
Likely cause: the request lacks `Authorization: Bearer <token>`, has an empty token, or uses a token that does not match any configured upload pipeline.
|
||||
Likely cause: the request lacks `Authorization: Bearer <token>`, has an empty token, or uses a token that does not match any configured upload token record.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
curl -i -X POST http://127.0.0.1:8080/upload \
|
||||
curl -i -X POST http://127.0.0.1:8080/v1/pipelines/<pipeline-id>/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/x-tar" \
|
||||
--data-binary @bundle.tar
|
||||
@@ -433,11 +545,27 @@ Safe fix: use the token value resolved by the configured `token_env`. Do not inc
|
||||
|
||||
Reference: [Operations](operations.md#http-upload-operation).
|
||||
|
||||
## Upload Request Is Forbidden
|
||||
|
||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `403`.
|
||||
|
||||
Likely cause: the bearer token is valid, but its configured `allow_pipelines` list does not include the requested upload pipeline.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
rg -n 'upload_tokens:|allow_pipelines:|id:' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: request the intended pipeline id, or update the token allowlist to include the configured `http_upload` pipeline that this producer may submit to.
|
||||
|
||||
Reference: [Configuration](config.md#upload_tokens).
|
||||
|
||||
## Upload Request Is Rejected Before A Run ID
|
||||
|
||||
Symptom: `POST /upload` returns `400`, `413`, `415`, or `503`.
|
||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `400`, `413`, `415`, or `503`.
|
||||
|
||||
Likely cause: the request included a `pipeline` or `pipeline_id` query, archive content is malformed, the body exceeds size limits, content type is unsupported, or the in-memory upload queue is full.
|
||||
Likely cause: the request path has an invalid pipeline id, included a `pipeline` or `pipeline_id` query, archive content is malformed, the body exceeds size limits, content type is unsupported, or the in-memory upload queue is full.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
@@ -447,20 +575,20 @@ tar -tzf bundle.tar.gz
|
||||
rg -n 'max_upload_size|queue_size|max_concurrency' <config-path>
|
||||
```
|
||||
|
||||
Safe fix: send one valid tar or tar.gz source bundle archive with `Content-Type: application/x-tar`, `application/gzip`, or `application/x-gzip`; remove pipeline query parameters; reduce archive size or raise the configured limit; retry after queue pressure drops.
|
||||
Safe fix: send one valid tar or tar.gz source bundle archive to `/v1/pipelines/<pipeline-id>/upload` with `Content-Type: application/x-tar`, `application/gzip`, or `application/x-gzip`; remove pipeline query parameters; reduce archive size or raise the configured limit; retry after queue pressure drops.
|
||||
|
||||
Reference: [Operations](operations.md#http-upload-operation).
|
||||
|
||||
## Upload Idempotency Conflict
|
||||
|
||||
Symptom: `POST /upload` returns `409`.
|
||||
Symptom: `POST /v1/pipelines/<pipeline-id>/upload` returns `409`.
|
||||
|
||||
Likely cause: the request reused an `Idempotency-Key` for the same authenticated pipeline with a different source manifest, or another request with the same key is still being staged before its manifest is known.
|
||||
Likely cause: the request reused an `Idempotency-Key` for the same token id and pipeline id with a different source manifest, or another request with the same key is still being staged before its manifest is known.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
curl -i -X POST http://127.0.0.1:8080/upload \
|
||||
curl -i -X POST http://127.0.0.1:8080/v1/pipelines/<pipeline-id>/upload \
|
||||
-H "Authorization: Bearer $DISTRIBUTOR_UPLOAD_TOKEN" \
|
||||
-H "Content-Type: application/gzip" \
|
||||
-H "Idempotency-Key: <key>" \
|
||||
@@ -469,7 +597,7 @@ curl -i -X POST http://127.0.0.1:8080/upload \
|
||||
|
||||
Safe fix: if the response includes `"retryable":true`, retry the same upload later with the same key. Otherwise, inspect the producer operation and use the same key only for the same source bundle.
|
||||
|
||||
Reference: [HTTP Upload API Contract](integrations/http-upload.md#post-upload).
|
||||
Reference: [HTTP Upload API Contract](integrations/http-upload.md).
|
||||
|
||||
## Upload Status Is Missing
|
||||
|
||||
|
||||
@@ -9,11 +9,15 @@ server:
|
||||
queue_size: 16
|
||||
max_concurrency: 1
|
||||
retention: 24h
|
||||
upload_tokens:
|
||||
- id: example-uploader
|
||||
token_env: DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN
|
||||
allow_pipelines:
|
||||
- example-http-upload
|
||||
pipelines:
|
||||
- id: example-http-upload
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: DISTRIBUTOR_EXAMPLE_UPLOAD_TOKEN
|
||||
destinations:
|
||||
- id: local-archive
|
||||
backend: local
|
||||
@@ -21,4 +25,3 @@ pipelines:
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
|
||||
|
||||
18
examples/merge-reconciliation.yml
Normal file
18
examples/merge-reconciliation.yml
Normal file
@@ -0,0 +1,18 @@
|
||||
pipelines:
|
||||
- id: example-merge-reconciliation
|
||||
source:
|
||||
backend: local
|
||||
path: examples/source-bundle
|
||||
destinations:
|
||||
- id: local-merge-html
|
||||
backend: local
|
||||
path: workspace/published/merge-reconciliation
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
reconciliation:
|
||||
mode: merge
|
||||
31
examples/shared-root.yml
Normal file
31
examples/shared-root.yml
Normal file
@@ -0,0 +1,31 @@
|
||||
pipelines:
|
||||
- id: example-shared-root-source
|
||||
source:
|
||||
backend: local
|
||||
path: examples/source-bundle
|
||||
destinations:
|
||||
- id: shared-local-root
|
||||
backend: local
|
||||
path: workspace/published/shared-root
|
||||
state:
|
||||
mode: shared_root
|
||||
publish:
|
||||
source: true
|
||||
html: false
|
||||
- id: example-shared-root-html
|
||||
source:
|
||||
backend: local
|
||||
path: examples/source-bundle
|
||||
destinations:
|
||||
- id: shared-local-root
|
||||
backend: local
|
||||
path: workspace/published/shared-root
|
||||
state:
|
||||
mode: shared_root
|
||||
publish:
|
||||
source: false
|
||||
html: true
|
||||
transform:
|
||||
markdown_to_html:
|
||||
enabled: true
|
||||
mode: sidecar
|
||||
@@ -23,6 +23,13 @@ func main() {
|
||||
if len(os.Args) > 1 {
|
||||
bundleRoot = os.Args[1]
|
||||
}
|
||||
pipelineID := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_PIPELINE_ID")
|
||||
if pipelineID == "" {
|
||||
pipelineID = "example-http-upload"
|
||||
}
|
||||
if len(os.Args) > 2 {
|
||||
pipelineID = os.Args[2]
|
||||
}
|
||||
idempotencyKey := os.Getenv("DISTRIBUTOR_EXAMPLE_UPLOAD_IDEMPOTENCY_KEY")
|
||||
|
||||
client, err := upload.NewClient(upload.ClientOptions{
|
||||
@@ -35,7 +42,10 @@ func main() {
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
opts := upload.UploadBundleOptions{Root: bundleRoot}
|
||||
opts := upload.UploadBundleOptions{
|
||||
PipelineID: pipelineID,
|
||||
Root: bundleRoot,
|
||||
}
|
||||
if idempotencyKey != "" {
|
||||
opts.IdempotencyKey = idempotencyKey
|
||||
}
|
||||
|
||||
@@ -199,10 +199,22 @@ func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
return b.deleteManagedTargets(ctx, storage.OpDeleteManagedBundle, func() ([]string, error) {
|
||||
return storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
}, opts)
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
return b.deleteManagedTargets(ctx, storage.OpDeleteManagedOutputs, func() ([]string, error) {
|
||||
return storage.ManagedOutputTargets(bundlePath, managedOutputPaths)
|
||||
}, opts)
|
||||
}
|
||||
|
||||
func (b *Backend) deleteManagedTargets(ctx context.Context, op string, targetsFunc func() ([]string, error), opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
targets, err := targetsFunc()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -213,20 +225,20 @@ func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, ma
|
||||
return err
|
||||
}
|
||||
if nativePath == b.root {
|
||||
return storage.NewError(storage.OpDeleteManagedBundle, backendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||
return storage.NewError(op, backendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||
}
|
||||
info, err := os.Lstat(nativePath)
|
||||
if err != nil {
|
||||
if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return b.translateError(storage.OpDeleteManagedBundle, logicalPath, err)
|
||||
return b.translateError(op, logicalPath, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return storage.NewError(storage.OpDeleteManagedBundle, backendName, logicalPath, storage.ErrUnsupported, nil)
|
||||
return storage.NewError(op, backendName, logicalPath, storage.ErrUnsupported, nil)
|
||||
}
|
||||
if err := os.Remove(nativePath); err != nil {
|
||||
return b.translateError(storage.OpDeleteManagedBundle, logicalPath, err)
|
||||
return b.translateError(op, logicalPath, err)
|
||||
}
|
||||
if opts.PruneEmptyDirs {
|
||||
b.pruneEmptyParents(filepath.Dir(nativePath))
|
||||
|
||||
@@ -205,15 +205,27 @@ func (b *Backend) HasAny(ctx context.Context, logicalPrefix string) (bool, error
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
return b.deleteManagedTargets(ctx, storage.OpDeleteManagedBundle, func() ([]string, error) {
|
||||
return storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
}, opts)
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
return b.deleteManagedTargets(ctx, storage.OpDeleteManagedOutputs, func() ([]string, error) {
|
||||
return storage.ManagedOutputTargets(bundlePath, managedOutputPaths)
|
||||
}, opts)
|
||||
}
|
||||
|
||||
func (b *Backend) deleteManagedTargets(ctx context.Context, op string, targetsFunc func() ([]string, error), opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
targets, err := targetsFunc()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, target := range targets {
|
||||
if err := b.deleteObject(ctx, storage.OpDeleteManagedBundle, target, opts); err != nil {
|
||||
if err := b.deleteObject(ctx, op, target, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,10 +223,22 @@ func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
return b.deleteManagedTargets(ctx, storage.OpDeleteManagedBundle, func() ([]string, error) {
|
||||
return storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
}, opts)
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
return b.deleteManagedTargets(ctx, storage.OpDeleteManagedOutputs, func() ([]string, error) {
|
||||
return storage.ManagedOutputTargets(bundlePath, managedOutputPaths)
|
||||
}, opts)
|
||||
}
|
||||
|
||||
func (b *Backend) deleteManagedTargets(ctx context.Context, op string, targetsFunc func() ([]string, error), opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
targets, err := targetsFunc()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -236,20 +248,20 @@ func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, ma
|
||||
return err
|
||||
}
|
||||
if nativePath == b.root {
|
||||
return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrInvalidPath, nil)
|
||||
return storage.NewError(op, BackendName, target, storage.ErrInvalidPath, nil)
|
||||
}
|
||||
info, err := b.client.Lstat(nativePath)
|
||||
if err != nil {
|
||||
if opts.IgnoreMissing && isNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return b.translateError(storage.OpDeleteManagedBundle, target, err)
|
||||
return b.translateError(op, target, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return storage.NewError(storage.OpDeleteManagedBundle, BackendName, target, storage.ErrUnsupported, nil)
|
||||
return storage.NewError(op, BackendName, target, storage.ErrUnsupported, nil)
|
||||
}
|
||||
if err := b.client.Remove(nativePath); err != nil {
|
||||
return b.translateError(storage.OpDeleteManagedBundle, target, err)
|
||||
return b.translateError(op, target, err)
|
||||
}
|
||||
if opts.PruneEmptyDirs {
|
||||
b.pruneEmptyParents(parentOf(target))
|
||||
|
||||
339
internal/app/prune.go
Normal file
339
internal/app/prune.go
Normal file
@@ -0,0 +1,339 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
type PruneOptions struct {
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
DryRun bool
|
||||
Now time.Time
|
||||
Stdout io.Writer
|
||||
OutputFormat OutputFormat
|
||||
}
|
||||
|
||||
type PrunePlanOptions struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type PrunePlanReport struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
OwnerScope PruneOwnerScope `json:"owner_scope"`
|
||||
Enabled bool `json:"enabled"`
|
||||
CheckedCount int `json:"checked_count"`
|
||||
PrunedOutputs []PruneOutputRecord `json:"pruned_outputs"`
|
||||
PreservedOutputs []PruneOutputRecord `json:"preserved_outputs"`
|
||||
}
|
||||
|
||||
type PruneReport struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Backend string `json:"backend"`
|
||||
RootPath string `json:"root_path"`
|
||||
OwnerScope PruneOwnerScope `json:"owner_scope"`
|
||||
Enabled bool `json:"enabled"`
|
||||
CheckedCount int `json:"checked_count"`
|
||||
PlannedOutputs []PruneOutputRecord `json:"planned_outputs"`
|
||||
DeletedOutputs []PruneOutputRecord `json:"deleted_outputs"`
|
||||
PreservedOutputs []PruneOutputRecord `json:"preserved_outputs"`
|
||||
FailedOutput *PruneOutputRecord `json:"failed_output,omitempty"`
|
||||
StateChanged bool `json:"state_changed"`
|
||||
WouldChange bool `json:"would_change"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
|
||||
type PruneOwnerScope struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
}
|
||||
|
||||
type PruneOutputRecord struct {
|
||||
Path string `json:"path"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
Owner *PruneOwnerScope `json:"owner,omitempty"`
|
||||
}
|
||||
|
||||
func Prune(ctx context.Context, options PruneOptions) (PruneReport, error) {
|
||||
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
|
||||
setup, err := loadRuntimeSetup(options.ConfigPath)
|
||||
if err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
return pruneSetup(ctx, setup, options)
|
||||
}
|
||||
|
||||
func pruneConfigWithBackendFactory(ctx context.Context, cfg config.Config, options PruneOptions, provider backendFactoryProvider) (PruneReport, error) {
|
||||
setup, err := runtimeSetupFromConfig("", cfg)
|
||||
if err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
return pruneSetupWithBackendFactory(ctx, setup, options, provider)
|
||||
}
|
||||
|
||||
func pruneSetup(ctx context.Context, setup runtimeSetup, options PruneOptions) (PruneReport, error) {
|
||||
return pruneSetupWithBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
|
||||
}
|
||||
|
||||
func pruneSetupWithBackendFactory(ctx context.Context, setup runtimeSetup, options PruneOptions, provider backendFactoryProvider) (PruneReport, error) {
|
||||
if err := requirePruneScope(options); err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
pipeline, ok := findPipeline(setup.Config, options.PipelineID)
|
||||
if !ok {
|
||||
return PruneReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
||||
}
|
||||
destination, ok := findDestination(pipeline, options.DestinationID)
|
||||
if !ok {
|
||||
return PruneReport{}, fmt.Errorf("pipeline %s destination %s not found", options.PipelineID, options.DestinationID)
|
||||
}
|
||||
|
||||
backends := provider(setup.Environment)
|
||||
destinationBackend, err := backends.openDestination(ctx, destination)
|
||||
if err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
defer closeBackend(destinationBackend)
|
||||
|
||||
report, err := executePrune(ctx, destinationBackend, pipeline, destination, options)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := WritePruneReport(options.Stdout, options.OutputFormat, report); err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func requirePruneScope(options PruneOptions) error {
|
||||
if options.PipelineID == "" {
|
||||
return fmt.Errorf("pipeline id is required")
|
||||
}
|
||||
if options.DestinationID == "" {
|
||||
return fmt.Errorf("destination id is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func executePrune(ctx context.Context, backend storage.Backend, pipeline config.Pipeline, destination config.Destination, options PruneOptions) (PruneReport, error) {
|
||||
now := options.Now
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
} else {
|
||||
now = now.UTC()
|
||||
}
|
||||
|
||||
statePath, err := storage.StatePath("")
|
||||
if err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
data, err := backend.ReadFile(ctx, statePath)
|
||||
if err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
document, err := state.ParseDocument(data)
|
||||
if err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
|
||||
plan, err := PlanPrune(document, destination.Retention.Prune, PrunePlanOptions{
|
||||
PipelineID: pipeline.ID,
|
||||
DestinationID: destination.ID,
|
||||
Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
report := PruneReport{
|
||||
PipelineID: pipeline.ID,
|
||||
DestinationID: destination.ID,
|
||||
Backend: destination.Backend,
|
||||
RootPath: destinationRootPath(destination),
|
||||
OwnerScope: plan.OwnerScope,
|
||||
Enabled: plan.Enabled,
|
||||
CheckedCount: plan.CheckedCount,
|
||||
PlannedOutputs: plan.PrunedOutputs,
|
||||
DeletedOutputs: []PruneOutputRecord{},
|
||||
PreservedOutputs: plan.PreservedOutputs,
|
||||
DryRun: options.DryRun,
|
||||
}
|
||||
report.WouldChange = options.DryRun && len(report.PlannedOutputs) > 0
|
||||
if options.DryRun || len(report.PlannedOutputs) == 0 {
|
||||
return report, nil
|
||||
}
|
||||
|
||||
deletedPaths := make([]string, 0, len(report.PlannedOutputs))
|
||||
for _, output := range report.PlannedOutputs {
|
||||
err := backend.DeleteManagedOutputs(ctx, "", []string{output.Path}, storage.DeleteOptions{
|
||||
IgnoreMissing: true,
|
||||
PruneEmptyDirs: true,
|
||||
})
|
||||
if err != nil {
|
||||
failed := output
|
||||
report.FailedOutput = &failed
|
||||
if len(deletedPaths) > 0 {
|
||||
changed, writeErr := removePrunedStateRecords(ctx, backend, statePath, document, state.CurrentOwnerScope(pipeline.ID, destination.ID), deletedPaths, now)
|
||||
report.StateChanged = changed
|
||||
report.DeletedOutputs = report.PlannedOutputs[:len(deletedPaths)]
|
||||
if writeErr != nil {
|
||||
return report, writeErr
|
||||
}
|
||||
}
|
||||
return report, err
|
||||
}
|
||||
deletedPaths = append(deletedPaths, output.Path)
|
||||
}
|
||||
|
||||
changed, err := removePrunedStateRecords(ctx, backend, statePath, document, state.CurrentOwnerScope(pipeline.ID, destination.ID), deletedPaths, now)
|
||||
report.StateChanged = changed
|
||||
report.DeletedOutputs = report.PlannedOutputs
|
||||
return report, err
|
||||
}
|
||||
|
||||
func removePrunedStateRecords(ctx context.Context, backend storage.Backend, statePath string, document state.StateDocument, scope state.OwnerScope, paths []string, now time.Time) (bool, error) {
|
||||
if len(paths) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
if document.SingleOwner != nil {
|
||||
next, changed := state.RemoveMissingOutputs(*document.SingleOwner, paths)
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
next.UpdatedAt = now
|
||||
if err := state.Validate(next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, writeRepairedState(ctx, backend, statePath, next)
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
next, changed := state.RemoveMissingSharedRootOwnerOutputs(*document.SharedRoot, scope, paths)
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
next.UpdatedAt = now
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, writeRepairedState(ctx, backend, statePath, next)
|
||||
}
|
||||
return false, fmt.Errorf("destination state document is empty")
|
||||
}
|
||||
|
||||
func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options PrunePlanOptions) (PrunePlanReport, error) {
|
||||
scope := state.CurrentOwnerScope(options.PipelineID, options.DestinationID)
|
||||
report := PrunePlanReport{
|
||||
PipelineID: options.PipelineID,
|
||||
DestinationID: options.DestinationID,
|
||||
OwnerScope: PruneOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID},
|
||||
Enabled: policy.Enabled,
|
||||
PrunedOutputs: []PruneOutputRecord{},
|
||||
PreservedOutputs: []PruneOutputRecord{},
|
||||
}
|
||||
if !policy.Enabled {
|
||||
return report, nil
|
||||
}
|
||||
|
||||
candidates, err := pruneCandidatesForDocument(document, scope)
|
||||
if err != nil {
|
||||
return PrunePlanReport{}, err
|
||||
}
|
||||
report.CheckedCount = len(candidates)
|
||||
plan := state.PlanPrune(candidates, state.PrunePlanOptions{
|
||||
Now: options.Now,
|
||||
OlderThan: pruneOlderThan(policy),
|
||||
KeepLatest: policy.KeepLatest,
|
||||
})
|
||||
report.PrunedOutputs = pruneOutputRecords(plan.Pruned)
|
||||
report.PreservedOutputs = pruneOutputRecords(plan.Preserved)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func pruneCandidatesForDocument(document state.StateDocument, scope state.OwnerScope) ([]state.PruneCandidate, error) {
|
||||
if document.SingleOwner != nil {
|
||||
singleOwner := *document.SingleOwner
|
||||
if singleOwner.PipelineID != scope.PipelineID || singleOwner.DestinationID != scope.DestinationID {
|
||||
return nil, fmt.Errorf("state owner is %s/%s, not %s/%s", singleOwner.PipelineID, singleOwner.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
return state.SingleOwnerPruneCandidates(singleOwner), nil
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
return state.SharedRootPruneCandidates(*document.SharedRoot, scope), nil
|
||||
}
|
||||
return nil, fmt.Errorf("destination state document is empty")
|
||||
}
|
||||
|
||||
func pruneOlderThan(policy config.PrunePolicy) *time.Duration {
|
||||
if policy.OlderThan == nil {
|
||||
return nil
|
||||
}
|
||||
duration := policy.OlderThan.AsDuration()
|
||||
return &duration
|
||||
}
|
||||
|
||||
func pruneOutputRecords(candidates []state.PruneCandidate) []PruneOutputRecord {
|
||||
records := make([]PruneOutputRecord, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
var owner *PruneOwnerScope
|
||||
if candidate.Owner != nil {
|
||||
owner = &PruneOwnerScope{
|
||||
PipelineID: candidate.Owner.PipelineID,
|
||||
DestinationID: candidate.Owner.DestinationID,
|
||||
}
|
||||
}
|
||||
records = append(records, PruneOutputRecord{
|
||||
Path: candidate.Path,
|
||||
UpdatedAt: candidate.UpdatedAt.UTC().Format(time.RFC3339),
|
||||
Owner: owner,
|
||||
})
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func WritePruneReport(w io.Writer, format OutputFormat, report PruneReport) error {
|
||||
if IsJSONOutput(format) {
|
||||
return WriteJSONEnvelope(w, "prune", true, nil, report, nil)
|
||||
}
|
||||
return writePruneReportText(w, report)
|
||||
}
|
||||
|
||||
func writePruneReportText(w io.Writer, report PruneReport) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
status := "unchanged"
|
||||
if report.StateChanged {
|
||||
status = "changed"
|
||||
} else if report.WouldChange {
|
||||
status = "would_change"
|
||||
}
|
||||
_, err := fmt.Fprintf(w, "Prune: pipeline=%s destination=%s backend=%s root=%s status=%s checked=%d planned=%d deleted=%d preserved=%d dry_run=%t\n",
|
||||
report.PipelineID,
|
||||
report.DestinationID,
|
||||
report.Backend,
|
||||
report.RootPath,
|
||||
status,
|
||||
report.CheckedCount,
|
||||
len(report.PlannedOutputs),
|
||||
len(report.DeletedOutputs),
|
||||
len(report.PreservedOutputs),
|
||||
report.DryRun,
|
||||
)
|
||||
return err
|
||||
}
|
||||
353
internal/app/prune_test.go
Normal file
353
internal/app/prune_test.go
Normal file
@@ -0,0 +1,353 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestPlanPruneDisabledPolicy(t *testing.T) {
|
||||
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
|
||||
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PlanPrune() error = %v", err)
|
||||
}
|
||||
if report.Enabled || report.CheckedCount != 0 || len(report.PrunedOutputs) != 0 {
|
||||
t.Fatalf("report = %#v, want disabled empty plan", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
olderThan := config.Duration(48 * time.Hour)
|
||||
destinationState := pruneSingleOwnerState(now)
|
||||
|
||||
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
|
||||
Enabled: true,
|
||||
OlderThan: &olderThan,
|
||||
}, PrunePlanOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PlanPrune() error = %v", err)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.PreservedOutputs), "fresh.txt"; got != want {
|
||||
t.Fatalf("preserved = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
keepLatest := 0
|
||||
sharedRoot := pruneSharedRootState(now)
|
||||
|
||||
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
|
||||
Enabled: true,
|
||||
KeepLatest: &keepLatest,
|
||||
}, PrunePlanOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Now: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PlanPrune() error = %v", err)
|
||||
}
|
||||
if got, want := report.CheckedCount, 1; got != want {
|
||||
t.Fatalf("checked count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
original := pruneSingleOwnerState(now)
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, original)
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
DryRun: true,
|
||||
Now: now,
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
||||
if err != nil {
|
||||
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.WouldChange || report.StateChanged || len(report.DeletedOutputs) != 0 {
|
||||
t.Fatalf("report would_change=%t state_changed=%t deleted=%d, want dry-run only", report.WouldChange, report.StateChanged, len(report.DeletedOutputs))
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.PlannedOutputs), "old.txt"; got != want {
|
||||
t.Fatalf("planned outputs = %q, want %q", got, want)
|
||||
}
|
||||
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
|
||||
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Now: now,
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
||||
if err != nil {
|
||||
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.StateChanged || report.WouldChange {
|
||||
t.Fatalf("report state_changed=%t would_change=%t, want applied change", report.StateChanged, report.WouldChange)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
||||
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
assertFakeStateExists(t, backend)
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
||||
t.Fatalf("state outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
if !destinationState.UpdatedAt.Equal(now) {
|
||||
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
keepLatest := 0
|
||||
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
||||
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Now: now,
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": failingBackend}))
|
||||
if err == nil {
|
||||
t.Fatal("pruneConfigWithBackendFactory() error = nil, want delete failure")
|
||||
}
|
||||
if report.FailedOutput == nil || report.FailedOutput.Path != "fresh.txt" {
|
||||
t.Fatalf("failed output = %#v, want fresh.txt", report.FailedOutput)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
||||
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
assertFakeStateExists(t, backend)
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
||||
t.Fatalf("state outputs = %q, want only failed output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
keepLatest := 0
|
||||
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
||||
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Now: now,
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
||||
if err != nil {
|
||||
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
|
||||
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "archive.txt")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "old")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
sharedRoot := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
|
||||
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func pruneSingleOwnerState(now time.Time) state.DistributorState {
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
publishedAt := now.Add(-96 * time.Hour)
|
||||
return state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
DistributorVersion: "test",
|
||||
Outputs: []state.OutputFile{{
|
||||
Path: "old.txt",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}, {
|
||||
Path: "fresh.txt",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
CreatedAt: now.Add(-24 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func pruneSharedRootState(now time.Time) state.SharedRootState {
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
archive := state.CurrentOwnerScope("reports", "archive")
|
||||
html := state.CurrentOwnerScope("reports", "html")
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: archive,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
}, {
|
||||
Scope: html,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "archive.txt",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
Owner: archive,
|
||||
SourceID: manifest.ID,
|
||||
SourceDigest: manifest.Digest,
|
||||
SourceCreated: manifest.Created,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}, {
|
||||
Path: "html.txt",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
Owner: html,
|
||||
SourceID: manifest.ID,
|
||||
SourceDigest: manifest.Digest,
|
||||
SourceCreated: manifest.Created,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func pruneRecordPaths(records []PruneOutputRecord) string {
|
||||
paths := make([]string, 0, len(records))
|
||||
for _, record := range records {
|
||||
paths = append(paths, record.Path)
|
||||
}
|
||||
return strings.Join(paths, ",")
|
||||
}
|
||||
|
||||
func pruneS3Config(t *testing.T, policy config.PrunePolicy) config.Config {
|
||||
t.Helper()
|
||||
cfg := config.Config{Pipelines: []config.Pipeline{{
|
||||
ID: "reports",
|
||||
Source: config.Backend{Backend: config.BackendLocal, Path: t.TempDir()},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "archive",
|
||||
Backend: config.BackendS3,
|
||||
Bucket: "reports",
|
||||
Retention: config.RetentionPolicy{
|
||||
Prune: policy,
|
||||
},
|
||||
}},
|
||||
}}}
|
||||
config.ApplyDefaults(&cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
|
||||
value := config.Duration(duration)
|
||||
return config.PrunePolicy{
|
||||
Enabled: true,
|
||||
OlderThan: &value,
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal single-owner state: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
||||
for _, output := range destinationState.Outputs {
|
||||
testutil.WriteFakeFile(t, backend, output.Path, "managed")
|
||||
}
|
||||
}
|
||||
|
||||
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
|
||||
t.Helper()
|
||||
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {
|
||||
t.Fatalf("state file stat error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type failingDeleteBackend struct {
|
||||
storage.Backend
|
||||
failPath string
|
||||
}
|
||||
|
||||
func (b failingDeleteBackend) DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
for _, path := range managedOutputPaths {
|
||||
if path == b.failPath {
|
||||
return storage.NewError(storage.OpDeleteManagedOutputs, "fake", path, storage.ErrPermission, nil)
|
||||
}
|
||||
}
|
||||
return b.Backend.DeleteManagedOutputs(ctx, bundlePath, managedOutputPaths, opts)
|
||||
}
|
||||
395
internal/app/reconcile_state.go
Normal file
395
internal/app/reconcile_state.go
Normal file
@@ -0,0 +1,395 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
const reconcileStateWalkLimit = 10000
|
||||
|
||||
type ReconcileStateOptions struct {
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
AllOwners bool
|
||||
DryRun bool
|
||||
Stdout io.Writer
|
||||
OutputFormat OutputFormat
|
||||
}
|
||||
|
||||
type ReconcileStateReport struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Backend string `json:"backend"`
|
||||
RootPath string `json:"root_path"`
|
||||
StateSchema int `json:"state_schema"`
|
||||
OwnerScope *ReconcileStateOwnerScope `json:"owner_scope,omitempty"`
|
||||
CheckedCount int `json:"checked_count"`
|
||||
MissingManagedOutputs []ReconcileStatePath `json:"missing_managed_outputs"`
|
||||
UnmanagedEntries []ReconcileStateEntry `json:"unmanaged_entries"`
|
||||
Changed bool `json:"changed"`
|
||||
WouldChange bool `json:"would_change"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
|
||||
type ReconcileStateOwnerScope struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
AllOwners bool `json:"all_owners,omitempty"`
|
||||
}
|
||||
|
||||
type ReconcileStatePath struct {
|
||||
Path string `json:"path"`
|
||||
OwnerScope *ReconcileStateOwnerScope `json:"owner_scope,omitempty"`
|
||||
StorageStatus string `json:"storage_status"`
|
||||
}
|
||||
|
||||
type ReconcileStateEntry struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
}
|
||||
|
||||
func ReconcileState(ctx context.Context, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
|
||||
setup, err := loadRuntimeSetup(options.ConfigPath)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
return reconcileStateSetup(ctx, setup, options)
|
||||
}
|
||||
|
||||
func reconcileStateConfigWithBackendFactory(ctx context.Context, cfg config.Config, options ReconcileStateOptions, provider backendFactoryProvider) (ReconcileStateReport, error) {
|
||||
setup, err := runtimeSetupFromConfig("", cfg)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
return reconcileStateSetupWithBackendFactory(ctx, setup, options, provider)
|
||||
}
|
||||
|
||||
func reconcileStateSetup(ctx context.Context, setup runtimeSetup, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
return reconcileStateSetupWithBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
|
||||
}
|
||||
|
||||
func reconcileStateSetupWithBackendFactory(ctx context.Context, setup runtimeSetup, options ReconcileStateOptions, provider backendFactoryProvider) (ReconcileStateReport, error) {
|
||||
if err := requireReconcileStateScope(options); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
pipeline, ok := findPipeline(setup.Config, options.PipelineID)
|
||||
if !ok {
|
||||
return ReconcileStateReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
||||
}
|
||||
destination, ok := findDestination(pipeline, options.DestinationID)
|
||||
if !ok {
|
||||
return ReconcileStateReport{}, fmt.Errorf("pipeline %s destination %s not found", options.PipelineID, options.DestinationID)
|
||||
}
|
||||
|
||||
backends := provider(setup.Environment)
|
||||
destinationBackend, err := backends.openDestination(ctx, destination)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
defer closeBackend(destinationBackend)
|
||||
|
||||
report, err := buildReconcileStateReport(ctx, destinationBackend, pipeline, destination, options)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := WriteReconcileStateReport(options.Stdout, options.OutputFormat, report); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func requireReconcileStateScope(options ReconcileStateOptions) error {
|
||||
if options.PipelineID == "" {
|
||||
return fmt.Errorf("pipeline id is required")
|
||||
}
|
||||
if options.DestinationID == "" {
|
||||
return fmt.Errorf("destination id is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func findDestination(pipeline config.Pipeline, id string) (config.Destination, bool) {
|
||||
for _, destination := range pipeline.Destinations {
|
||||
if destination.ID == id {
|
||||
return destination, true
|
||||
}
|
||||
}
|
||||
return config.Destination{}, false
|
||||
}
|
||||
|
||||
func buildReconcileStateReport(ctx context.Context, backend storage.Backend, pipeline config.Pipeline, destination config.Destination, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
statePath, err := storage.StatePath("")
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
data, err := backend.ReadFile(ctx, statePath)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
document, err := state.ParseDocument(data)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
|
||||
report := ReconcileStateReport{
|
||||
PipelineID: pipeline.ID,
|
||||
DestinationID: destination.ID,
|
||||
Backend: destination.Backend,
|
||||
RootPath: destinationRootPath(destination),
|
||||
MissingManagedOutputs: []ReconcileStatePath{},
|
||||
UnmanagedEntries: []ReconcileStateEntry{},
|
||||
DryRun: options.DryRun,
|
||||
}
|
||||
scope := state.CurrentOwnerScope(pipeline.ID, destination.ID)
|
||||
if document.SingleOwner != nil {
|
||||
return reconcileSingleOwnerState(ctx, backend, statePath, *document.SingleOwner, scope, report, options)
|
||||
}
|
||||
return reconcileSharedRootState(ctx, backend, statePath, *document.SharedRoot, scope, report, options)
|
||||
}
|
||||
|
||||
func reconcileSingleOwnerState(ctx context.Context, backend storage.Backend, statePath string, destinationState state.DistributorState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
if destinationState.PipelineID != scope.PipelineID || destinationState.DestinationID != scope.DestinationID {
|
||||
return ReconcileStateReport{}, fmt.Errorf("state owner is %s/%s, not %s/%s", destinationState.PipelineID, destinationState.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
report.StateSchema = destinationState.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID}
|
||||
managed := state.ManagedOutputPaths(destinationState)
|
||||
missing, err := missingSingleOwnerOutputs(ctx, backend, destinationState.Outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.CheckedCount = len(managed)
|
||||
report.MissingManagedOutputs = missing
|
||||
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.UnmanagedEntries = unmanaged
|
||||
report.WouldChange = options.DryRun && len(missing) > 0
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
next, changed := state.RemoveMissingOutputs(destinationState, missingPaths)
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.Validate(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func reconcileSharedRootState(ctx context.Context, backend storage.Backend, statePath string, sharedRoot state.SharedRootState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
report.StateSchema = sharedRoot.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{
|
||||
PipelineID: scope.PipelineID,
|
||||
DestinationID: scope.DestinationID,
|
||||
AllOwners: options.AllOwners,
|
||||
}
|
||||
managed := sharedRoot.AllManagedOutputPaths()
|
||||
outputs := sharedRoot.Outputs
|
||||
if !options.AllOwners {
|
||||
outputs = sharedRootOutputsForOwner(sharedRoot.Outputs, scope)
|
||||
}
|
||||
missing, err := missingSharedRootOutputs(ctx, backend, outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.CheckedCount = len(outputs)
|
||||
report.MissingManagedOutputs = missing
|
||||
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.UnmanagedEntries = unmanaged
|
||||
report.WouldChange = options.DryRun && len(missing) > 0
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
var next state.SharedRootState
|
||||
var changed bool
|
||||
if options.AllOwners {
|
||||
next, changed = state.RemoveMissingSharedRootOutputs(sharedRoot, missingPaths)
|
||||
} else {
|
||||
next, changed = state.RemoveMissingSharedRootOwnerOutputs(sharedRoot, scope, missingPaths)
|
||||
}
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func missingSingleOwnerOutputs(ctx context.Context, backend storage.Backend, outputs []state.OutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
if storage.IsNotFound(err) {
|
||||
missing = append(missing, ReconcileStatePath{Path: output.Path, StorageStatus: "missing"})
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outputs []state.SharedRootOutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
if storage.IsNotFound(err) {
|
||||
missing = append(missing, ReconcileStatePath{
|
||||
Path: output.Path,
|
||||
OwnerScope: &ReconcileStateOwnerScope{
|
||||
PipelineID: output.Owner.PipelineID,
|
||||
DestinationID: output.Owner.DestinationID,
|
||||
},
|
||||
StorageStatus: "missing",
|
||||
})
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func checkManagedOutput(ctx context.Context, backend storage.Backend, path string) error {
|
||||
_, err := backend.Stat(ctx, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func unmanagedEntries(ctx context.Context, backend storage.Backend, managedPaths []string) ([]ReconcileStateEntry, error) {
|
||||
managed := make(map[string]struct{}, len(managedPaths)+1)
|
||||
for _, path := range managedPaths {
|
||||
managed[path] = struct{}{}
|
||||
}
|
||||
managed[storage.StateFileName] = struct{}{}
|
||||
|
||||
entries := make([]ReconcileStateEntry, 0)
|
||||
err := backend.Walk(ctx, "", storage.WalkOptions{Recursive: true, Limit: reconcileStateWalkLimit}, func(entry storage.Entry) error {
|
||||
if entry.Type == storage.EntryTypeDirectory {
|
||||
return nil
|
||||
}
|
||||
if _, ok := managed[entry.Path]; ok {
|
||||
return nil
|
||||
}
|
||||
entries = append(entries, ReconcileStateEntry{
|
||||
Path: entry.Path,
|
||||
Type: string(entry.Type),
|
||||
Size: entry.Size,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return entries[i].Path < entries[j].Path
|
||||
})
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func sharedRootOutputsForOwner(outputs []state.SharedRootOutputFile, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
selected := make([]state.SharedRootOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if output.Owner == scope {
|
||||
selected = append(selected, output)
|
||||
}
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func missingReportPaths(missing []ReconcileStatePath) []string {
|
||||
paths := make([]string, 0, len(missing))
|
||||
for _, item := range missing {
|
||||
paths = append(paths, item.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func writeRepairedState(ctx context.Context, backend storage.Backend, path string, value any) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
_, err = backend.WriteFile(ctx, path, data, storage.WriteOptions{Overwrite: true, PreferAtomic: true})
|
||||
return err
|
||||
}
|
||||
|
||||
func destinationRootPath(destination config.Destination) string {
|
||||
switch destination.Backend {
|
||||
case config.BackendS3:
|
||||
if destination.Prefix == "" {
|
||||
return "."
|
||||
}
|
||||
return destination.Prefix
|
||||
default:
|
||||
if destination.Path == "" {
|
||||
return "."
|
||||
}
|
||||
return destination.Path
|
||||
}
|
||||
}
|
||||
|
||||
func WriteReconcileStateReport(w io.Writer, format OutputFormat, report ReconcileStateReport) error {
|
||||
if IsJSONOutput(format) {
|
||||
return WriteJSONEnvelope(w, "reconcile-state", true, nil, report, nil)
|
||||
}
|
||||
return writeReconcileStateReportText(w, report)
|
||||
}
|
||||
|
||||
func writeReconcileStateReportText(w io.Writer, report ReconcileStateReport) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
status := "unchanged"
|
||||
if report.Changed {
|
||||
status = "changed"
|
||||
} else if report.WouldChange {
|
||||
status = "would_change"
|
||||
}
|
||||
_, err := fmt.Fprintf(w, "Reconcile state: pipeline=%s destination=%s backend=%s root=%s status=%s checked=%d missing=%d unmanaged=%d dry_run=%t\n",
|
||||
report.PipelineID,
|
||||
report.DestinationID,
|
||||
report.Backend,
|
||||
report.RootPath,
|
||||
status,
|
||||
report.CheckedCount,
|
||||
len(report.MissingManagedOutputs),
|
||||
len(report.UnmanagedEntries),
|
||||
report.DryRun,
|
||||
)
|
||||
return err
|
||||
}
|
||||
269
internal/app/reconcile_state_test.go
Normal file
269
internal/app/reconcile_state_test.go
Normal file
@@ -0,0 +1,269 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed output: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
|
||||
report, err := reconcileStateConfigWithBackendFactory(context.Background(), cfg, ReconcileStateOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
DryRun: true,
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.WouldChange || report.Changed {
|
||||
t.Fatalf("report changed=%t would_change=%t, want dry-run pending change", report.Changed, report.WouldChange)
|
||||
}
|
||||
if got := reportPathList(report.MissingManagedOutputs); got != "summary.txt" {
|
||||
t.Fatalf("missing outputs = %q, want summary.txt", got)
|
||||
}
|
||||
if got := entryPathList(report.UnmanagedEntries); got != "extra.txt" {
|
||||
t.Fatalf("unmanaged entries = %q, want extra.txt", got)
|
||||
}
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed output: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
|
||||
report, err := reconcileStateConfigWithBackendFactory(context.Background(), cfg, ReconcileStateOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.Changed || report.WouldChange {
|
||||
t.Fatalf("report changed=%t would_change=%t, want applied change", report.Changed, report.WouldChange)
|
||||
}
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if err := state.Validate(destinationState); err != nil {
|
||||
t.Fatalf("Validate() repaired state error = %v", err)
|
||||
}
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
|
||||
t.Fatalf("state outputs = %q, want report.md", got)
|
||||
}
|
||||
testutil.AssertFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
}
|
||||
|
||||
func TestReconcileStateInvalidStateFailsWithoutRewrite(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
invalid := `{"schema_version":2,"pipeline_id":"reports"}`
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, invalid)
|
||||
|
||||
_, err := reconcileStateConfigWithBackendFactory(context.Background(), cfg, ReconcileStateOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
||||
if err == nil {
|
||||
t.Fatal("reconcileStateConfigWithBackendFactory() error = nil, want invalid state error")
|
||||
}
|
||||
data, readErr := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read invalid state: %v", readErr)
|
||||
}
|
||||
if string(data) != invalid {
|
||||
t.Fatalf("state data = %q, want original invalid data", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
sharedRoot := reconcileSharedRootFixture(t)
|
||||
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed outputs: %v", err)
|
||||
}
|
||||
|
||||
report, err := reconcileStateConfigWithBackendFactory(context.Background(), cfg, ReconcileStateOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.Changed {
|
||||
t.Fatal("report changed = false, want true")
|
||||
}
|
||||
repaired := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := strings.Join(repaired.AllManagedOutputPaths(), ","); got != "report.html" {
|
||||
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
sharedRoot := reconcileSharedRootFixture(t)
|
||||
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed outputs: %v", err)
|
||||
}
|
||||
|
||||
report, err := reconcileStateConfigWithBackendFactory(context.Background(), cfg, ReconcileStateOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
AllOwners: true,
|
||||
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.Changed || report.CheckedCount != 2 {
|
||||
t.Fatalf("report changed=%t checked=%d, want all-owner repair", report.Changed, report.CheckedCount)
|
||||
}
|
||||
repaired := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := repaired.AllManagedOutputPaths(); len(got) != 0 {
|
||||
t.Fatalf("shared-root outputs = %#v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func reconcileStateS3Config(t *testing.T) config.Config {
|
||||
t.Helper()
|
||||
cfg := config.Config{Pipelines: []config.Pipeline{{
|
||||
ID: "reports",
|
||||
Source: config.Backend{Backend: config.BackendLocal, Path: t.TempDir()},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "archive",
|
||||
Backend: config.BackendS3,
|
||||
Bucket: "reports",
|
||||
}},
|
||||
}}}
|
||||
config.ApplyDefaults(&cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func readFakeSingleOwnerState(t *testing.T, backend *fake.Backend) state.DistributorState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func writeFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend, sharedRoot state.SharedRootState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(sharedRoot, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared-root state: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
testutil.WriteFakeFile(t, backend, output.Path, "old")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend) state.SharedRootState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
}
|
||||
sharedRoot, err := state.ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
}
|
||||
return sharedRoot
|
||||
}
|
||||
|
||||
func reconcileSharedRootFixture(t *testing.T) state.SharedRootState {
|
||||
t.Helper()
|
||||
source := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
htmlSource := source
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: state.CurrentOwnerScope("reports", "archive"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: source},
|
||||
}, {
|
||||
Scope: state.CurrentOwnerScope("reports", "html"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
||||
Source: state.SourceState{Manifest: htmlSource},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
Owner: state.CurrentOwnerScope("reports", "archive"),
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
Kind: state.OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
SHA256: "sha256:" + strings.Repeat("a", 64),
|
||||
Size: 128,
|
||||
Owner: state.CurrentOwnerScope("reports", "html"),
|
||||
SourceID: htmlSource.ID,
|
||||
SourceDigest: htmlSource.Digest,
|
||||
SourceCreated: htmlSource.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func reportPathList(paths []ReconcileStatePath) string {
|
||||
values := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
values = append(values, path.Path)
|
||||
}
|
||||
return strings.Join(values, ",")
|
||||
}
|
||||
|
||||
func entryPathList(entries []ReconcileStateEntry) string {
|
||||
values := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
values = append(values, entry.Path)
|
||||
}
|
||||
return strings.Join(values, ",")
|
||||
}
|
||||
@@ -69,6 +69,8 @@ func processDestinationSelection(ctx context.Context, request runDestinationRequ
|
||||
Publish: *request.destination.Publish,
|
||||
Transform: request.destination.Transform,
|
||||
Links: request.destination.Links,
|
||||
State: request.destination.State,
|
||||
Reconciliation: request.destination.Reconciliation,
|
||||
Transformers: request.transforms,
|
||||
Transfer: request.destination.Transfer,
|
||||
DistributorVersion: Version,
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
@@ -221,6 +222,67 @@ func TestRunPublishesNewLocalBundle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSharedRootDryRunWritesNoOutputsOrState(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, sourceRoot, "", testBundleOptions{
|
||||
Files: []testFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
configPath := writeSharedRootLocalConfig(t, sourceRoot, destinationRoot)
|
||||
|
||||
cfg, err := config.LoadFile(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
report, err := buildRunReportWithBackendFactory(context.Background(), cfg, RunOptions{DryRun: true}, newBackendFactoryWithEnvironment)
|
||||
if err != nil {
|
||||
t.Fatalf("dry-run buildRunReportWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if got, want := report.Actions[0].Action, string(publish.ActionPublishNew); got != want {
|
||||
t.Fatalf("dry-run action = %q, want %q", got, want)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("output stat error = %v, want absent", statErr)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); !os.IsNotExist(statErr) {
|
||||
t.Fatalf("state file stat error = %v, want absent", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPublishesTwoPipelinesIntoSharedRoot(t *testing.T) {
|
||||
firstSourceRoot := t.TempDir()
|
||||
secondSourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
writeSourceBundle(t, firstSourceRoot, "", testBundleOptions{
|
||||
ID: "reports.first",
|
||||
Files: []testFile{{Path: "first.md", Data: "# First\n"}},
|
||||
})
|
||||
writeSourceBundle(t, secondSourceRoot, "", testBundleOptions{
|
||||
ID: "reports.second",
|
||||
Files: []testFile{{Path: "second.md", Data: "# Second\n"}},
|
||||
})
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: writeTwoPipelineSharedRootConfig(t, firstSourceRoot, secondSourceRoot, destinationRoot)})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "first.md"), "# First\n")
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "second.md"), "# Second\n")
|
||||
destinationState := readSharedRootStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got, want := len(destinationState.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-first", "archive")); !ok {
|
||||
t.Fatal("reports-first/archive owner missing")
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports-second", "archive")); !ok {
|
||||
t.Fatal("reports-second/archive owner missing")
|
||||
}
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "first.md,second.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineWithLocalSourcePublishesConfiguredDestination(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -292,7 +354,6 @@ func TestRunPipelineWithLocalSourcePublishesToRegisteredDestinationBackends(t *t
|
||||
ID: "reports",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{TokenEnv: "UPLOAD_TOKEN"},
|
||||
},
|
||||
Destinations: []config.Destination{
|
||||
{
|
||||
@@ -309,6 +370,11 @@ func TestRunPipelineWithLocalSourcePublishesToRegisteredDestinationBackends(t *t
|
||||
},
|
||||
},
|
||||
}},
|
||||
UploadTokens: []config.UploadToken{{
|
||||
ID: "reporter",
|
||||
TokenEnv: "UPLOAD_TOKEN",
|
||||
AllowPipelines: []string{"reports"},
|
||||
}},
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
|
||||
@@ -786,6 +852,53 @@ func TestRunNotifiesAfterReplacement(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMergeReconciliationRetainsManagedOutput(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
older := manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
defaultManifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
older.Files = append([]bundle.ManifestFile(nil), defaultManifest.Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
writeDestinationState(t, destinationRoot, "", older)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old report: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "summary.txt"), []byte("old summary\n"), 0o600); err != nil {
|
||||
t.Fatalf("write old summary: %v", err)
|
||||
}
|
||||
configPath := writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
reconciliation:
|
||||
mode: merge
|
||||
`)
|
||||
|
||||
err := Run(context.Background(), RunOptions{ConfigPath: configPath})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nNew.\n")
|
||||
testutil.AssertFile(t, filepath.Join(destinationRoot, "summary.txt"), "old summary\n")
|
||||
destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunJSONIncludesGeneratedOutputMetadata(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
@@ -1666,6 +1779,50 @@ func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
||||
return testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||
}
|
||||
|
||||
func writeSharedRootLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
`)
|
||||
}
|
||||
|
||||
func writeTwoPipelineSharedRootConfig(t *testing.T, firstSourceRoot, secondSourceRoot, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
pipelines:
|
||||
- id: reports-first
|
||||
source:
|
||||
backend: local
|
||||
path: `+firstSourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
- id: reports-second
|
||||
source:
|
||||
backend: local
|
||||
path: `+secondSourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
state:
|
||||
mode: shared_root
|
||||
`)
|
||||
}
|
||||
|
||||
func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestination string) string {
|
||||
t.Helper()
|
||||
return testutil.WriteFanoutLocalConfig(t, sourceRoot, firstDestination, secondDestination)
|
||||
@@ -1674,11 +1831,15 @@ func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestina
|
||||
func writeUploadPipelineConfig(t *testing.T, destinationRoot string) string {
|
||||
t.Helper()
|
||||
return writeConfigFile(t, `
|
||||
upload_tokens:
|
||||
- id: reporter
|
||||
token_env: UPLOAD_TOKEN
|
||||
allow_pipelines:
|
||||
- reports
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: UPLOAD_TOKEN
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
@@ -1742,6 +1903,19 @@ func readStateFile(t *testing.T, path string) state.DistributorState {
|
||||
return testutil.ReadDestinationState(t, path)
|
||||
}
|
||||
|
||||
func readSharedRootStateFile(t *testing.T, path string) state.SharedRootState {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
}
|
||||
destinationState, err := state.ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
|
||||
byPath := make(map[string]state.OutputFile, len(outputs))
|
||||
for _, output := range outputs {
|
||||
|
||||
@@ -70,14 +70,24 @@ func writeServeUploadConfig(t *testing.T, tokenEnvs []string) string {
|
||||
server:
|
||||
http:
|
||||
bind: 127.0.0.1:0
|
||||
pipelines:
|
||||
upload_tokens:
|
||||
`
|
||||
for index, tokenEnv := range tokenEnvs {
|
||||
body += `
|
||||
- id: reporter-` + string(rune('a'+index)) + `
|
||||
token_env: ` + tokenEnv + `
|
||||
allow_pipelines:
|
||||
- reports-` + string(rune('a'+index)) + `
|
||||
`
|
||||
}
|
||||
body += `
|
||||
pipelines:
|
||||
`
|
||||
for index := range tokenEnvs {
|
||||
body += `
|
||||
- id: reports-` + string(rune('a'+index)) + `
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: ` + tokenEnv + `
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
|
||||
@@ -44,6 +44,7 @@ type UploadRunRecord struct {
|
||||
}
|
||||
|
||||
type UploadRequest struct {
|
||||
TokenID string
|
||||
PipelineID string
|
||||
ContentType string
|
||||
Body io.Reader
|
||||
@@ -115,6 +116,7 @@ type uploadJob struct {
|
||||
}
|
||||
|
||||
type uploadIdempotencyScope struct {
|
||||
TokenID string
|
||||
PipelineID string
|
||||
Key string
|
||||
}
|
||||
@@ -200,7 +202,7 @@ func (coordinator *UploadCoordinator) Submit(ctx context.Context, request Upload
|
||||
if err := ingest.ValidateContentType(request.ContentType); err != nil {
|
||||
return UploadRunRecord{}, err
|
||||
}
|
||||
scope, hasKey := uploadRequestIdempotencyScope(pipeline.ID, request.IdempotencyKey)
|
||||
scope, hasKey := uploadRequestIdempotencyScope(request.TokenID, pipeline.ID, request.IdempotencyKey)
|
||||
|
||||
coordinator.mu.Lock()
|
||||
coordinator.expireLocked(coordinator.now().UTC())
|
||||
@@ -286,11 +288,11 @@ func (coordinator *UploadCoordinator) Submit(ctx context.Context, request Upload
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func uploadRequestIdempotencyScope(pipelineID, key string) (uploadIdempotencyScope, bool) {
|
||||
func uploadRequestIdempotencyScope(tokenID, pipelineID, key string) (uploadIdempotencyScope, bool) {
|
||||
if key == "" {
|
||||
return uploadIdempotencyScope{}, false
|
||||
}
|
||||
return uploadIdempotencyScope{PipelineID: pipelineID, Key: key}, true
|
||||
return uploadIdempotencyScope{TokenID: tokenID, PipelineID: pipelineID, Key: key}, true
|
||||
}
|
||||
|
||||
func (coordinator *UploadCoordinator) Status(runID UploadRunID) (UploadRunRecord, bool) {
|
||||
|
||||
@@ -271,6 +271,7 @@ func TestUploadCoordinatorIdempotencyReturnsOriginalRunForSameManifest(t *testin
|
||||
})
|
||||
|
||||
first, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("same"),
|
||||
@@ -282,6 +283,7 @@ func TestUploadCoordinatorIdempotencyReturnsOriginalRunForSameManifest(t *testin
|
||||
waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
|
||||
|
||||
second, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("same"),
|
||||
@@ -310,6 +312,7 @@ func TestUploadCoordinatorIdempotencyConflictsForDifferentManifest(t *testing.T)
|
||||
})
|
||||
|
||||
first, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("one"),
|
||||
@@ -321,6 +324,7 @@ func TestUploadCoordinatorIdempotencyConflictsForDifferentManifest(t *testing.T)
|
||||
waitForUploadStatus(t, coordinator, first.ID, UploadStatusSucceeded)
|
||||
|
||||
_, err = coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("two"),
|
||||
@@ -331,6 +335,42 @@ func TestUploadCoordinatorIdempotencyConflictsForDifferentManifest(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorIdempotencyIsScopedByToken(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
|
||||
pipelineIDs: []string{"reports"},
|
||||
}), uploadCoordinatorHooks{
|
||||
randomSuffix: uploadTestSuffixes("00000001", "00000002"),
|
||||
stage: manifestUploadStage,
|
||||
run: successfulUploadRun,
|
||||
})
|
||||
|
||||
first, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("one"),
|
||||
IdempotencyKey: "shared-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("first Submit() error = %v", err)
|
||||
}
|
||||
second, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-b",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("two"),
|
||||
IdempotencyKey: "shared-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("second Submit() error = %v", err)
|
||||
}
|
||||
if second.ID == first.ID {
|
||||
t.Fatalf("run ids matched across tokens: %q", second.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorIdempotencyIsScopedByPipeline(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -343,6 +383,7 @@ func TestUploadCoordinatorIdempotencyIsScopedByPipeline(t *testing.T) {
|
||||
})
|
||||
|
||||
first, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports-one",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("one"),
|
||||
@@ -352,6 +393,7 @@ func TestUploadCoordinatorIdempotencyIsScopedByPipeline(t *testing.T) {
|
||||
t.Fatalf("first Submit() error = %v", err)
|
||||
}
|
||||
second, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports-two",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("two"),
|
||||
@@ -408,6 +450,7 @@ func TestUploadCoordinatorIdempotencyReturnsRetryableConflictWhileStaging(t *tes
|
||||
firstErr := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("same"),
|
||||
@@ -419,6 +462,7 @@ func TestUploadCoordinatorIdempotencyReturnsRetryableConflictWhileStaging(t *tes
|
||||
|
||||
var reads atomic.Int64
|
||||
_, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: readerFunc(func(data []byte) (int, error) {
|
||||
@@ -440,6 +484,57 @@ func TestUploadCoordinatorIdempotencyReturnsRetryableConflictWhileStaging(t *tes
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorIdempotencyPendingScopeIncludesToken(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var calls atomic.Int64
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
coordinator := newUploadCoordinator(ctx, uploadCoordinatorConfig(t, uploadCoordinatorConfigOptions{
|
||||
pipelineIDs: []string{"reports"},
|
||||
}), uploadCoordinatorHooks{
|
||||
randomSuffix: uploadTestSuffixes("00000001", "00000002"),
|
||||
stage: func(ctx context.Context, opts ingest.StageOptions) (ingest.StagedBundle, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
close(entered)
|
||||
<-release
|
||||
}
|
||||
return manifestUploadStage(ctx, opts)
|
||||
},
|
||||
run: successfulUploadRun,
|
||||
})
|
||||
firstErr := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("same"),
|
||||
IdempotencyKey: "in-flight",
|
||||
})
|
||||
firstErr <- err
|
||||
}()
|
||||
<-entered
|
||||
|
||||
second, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-b",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("same"),
|
||||
IdempotencyKey: "in-flight",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("second Submit() error = %v", err)
|
||||
}
|
||||
if second.ID == "" {
|
||||
t.Fatal("second run id is empty, want accepted run")
|
||||
}
|
||||
close(release)
|
||||
if err := <-firstErr; err != nil {
|
||||
t.Fatalf("first Submit() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadCoordinatorIdempotencyExpiresWithCompletedStatus(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -455,6 +550,7 @@ func TestUploadCoordinatorIdempotencyExpiresWithCompletedStatus(t *testing.T) {
|
||||
})
|
||||
|
||||
first, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("same"),
|
||||
@@ -469,6 +565,7 @@ func TestUploadCoordinatorIdempotencyExpiresWithCompletedStatus(t *testing.T) {
|
||||
coordinator.Expire()
|
||||
|
||||
second, err := coordinator.Submit(context.Background(), UploadRequest{
|
||||
TokenID: "reporter-a",
|
||||
PipelineID: "reports",
|
||||
ContentType: ingest.ContentTypeTar,
|
||||
Body: strings.NewReader("same"),
|
||||
@@ -564,11 +661,11 @@ func uploadCoordinatorConfig(t *testing.T, opts uploadCoordinatorConfigOptions)
|
||||
}},
|
||||
}
|
||||
for _, pipelineID := range opts.pipelineIDs {
|
||||
tokenEnv := strings.ToUpper(strings.ReplaceAll(pipelineID, "-", "_")) + "_TOKEN"
|
||||
cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
|
||||
ID: pipelineID,
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{TokenEnv: strings.ToUpper(strings.ReplaceAll(pipelineID, "-", "_")) + "_TOKEN"},
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "archive",
|
||||
@@ -576,6 +673,11 @@ func uploadCoordinatorConfig(t *testing.T, opts uploadCoordinatorConfigOptions)
|
||||
Path: t.TempDir(),
|
||||
}},
|
||||
})
|
||||
cfg.UploadTokens = append(cfg.UploadTokens, config.UploadToken{
|
||||
ID: pipelineID + "-reporter",
|
||||
TokenEnv: tokenEnv,
|
||||
AllowPipelines: []string{pipelineID},
|
||||
})
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
@@ -20,7 +20,14 @@ type uploadCoordinator interface {
|
||||
|
||||
type uploadHTTPHandler struct {
|
||||
coordinator uploadCoordinator
|
||||
tokens map[string]string
|
||||
tokens map[string]resolvedUploadToken
|
||||
uploadPipelines map[string]struct{}
|
||||
}
|
||||
|
||||
type resolvedUploadToken struct {
|
||||
ID string
|
||||
Value string
|
||||
AllowedPipelines map[string]struct{}
|
||||
}
|
||||
|
||||
type uploadAcceptedResponse struct {
|
||||
@@ -44,36 +51,55 @@ func newUploadHTTPHandler(ctx context.Context, cfg config.Config, environment co
|
||||
return uploadHTTPHandler{
|
||||
coordinator: NewUploadCoordinator(ctx, cfg),
|
||||
tokens: tokens,
|
||||
uploadPipelines: uploadPipelineSet(cfg),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]string, error) {
|
||||
tokens := make(map[string]string)
|
||||
for _, pipeline := range cfg.Pipelines {
|
||||
if pipeline.Source.Backend != config.BackendHTTPUpload {
|
||||
continue
|
||||
}
|
||||
tokenName := pipeline.Source.Upload.TokenEnv
|
||||
token, ok := environment.Lookup(tokenName)
|
||||
func resolveUploadTokens(cfg config.Config, environment config.Environment) (map[string]resolvedUploadToken, error) {
|
||||
tokens := make(map[string]resolvedUploadToken)
|
||||
for _, uploadToken := range cfg.UploadTokens {
|
||||
token, ok := environment.Lookup(uploadToken.TokenEnv)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("upload token environment variable %s is not set", tokenName)
|
||||
return nil, fmt.Errorf("upload token %s environment variable %s is not set", uploadToken.ID, uploadToken.TokenEnv)
|
||||
}
|
||||
if token == "" {
|
||||
return nil, fmt.Errorf("upload token environment variable %s is empty", tokenName)
|
||||
return nil, fmt.Errorf("upload token %s environment variable %s is empty", uploadToken.ID, uploadToken.TokenEnv)
|
||||
}
|
||||
if existing, exists := tokens[token]; exists {
|
||||
return nil, fmt.Errorf("upload token environment variables for pipelines %s and %s resolve to the same value", existing, pipeline.ID)
|
||||
return nil, fmt.Errorf("upload token environment variables for tokens %s and %s resolve to the same value", existing.ID, uploadToken.ID)
|
||||
}
|
||||
tokens[token] = resolvedUploadToken{
|
||||
ID: uploadToken.ID,
|
||||
Value: token,
|
||||
AllowedPipelines: pipelineIDSet(uploadToken.AllowPipelines),
|
||||
}
|
||||
tokens[token] = pipeline.ID
|
||||
}
|
||||
return tokens, nil
|
||||
}
|
||||
|
||||
func uploadPipelineSet(cfg config.Config) map[string]struct{} {
|
||||
pipelines := make(map[string]struct{})
|
||||
for _, pipeline := range cfg.Pipelines {
|
||||
if pipeline.Source.Backend == config.BackendHTTPUpload {
|
||||
pipelines[pipeline.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
return pipelines
|
||||
}
|
||||
|
||||
func pipelineIDSet(ids []string) map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
set[id] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/healthz":
|
||||
handler.handleHealth(w)
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/upload":
|
||||
case r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/v1/pipelines/"):
|
||||
handler.handleUpload(w, r)
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/runs/"):
|
||||
handler.handleRunStatus(w, r)
|
||||
@@ -91,11 +117,28 @@ func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Req
|
||||
writeHTTPError(w, http.StatusBadRequest, "pipeline id is not accepted")
|
||||
return
|
||||
}
|
||||
pipelineID, ok := handler.authenticate(r.Header.Get("Authorization"))
|
||||
pipelineID, ok := uploadPipelineIDFromPath(r.URL.Path)
|
||||
if !ok {
|
||||
writeHTTPError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
if !config.IsSlugLikeID(pipelineID) {
|
||||
writeHTTPError(w, http.StatusBadRequest, "invalid pipeline id")
|
||||
return
|
||||
}
|
||||
token, ok := handler.authenticate(r.Header.Get("Authorization"))
|
||||
if !ok {
|
||||
writeHTTPError(w, http.StatusUnauthorized, "unauthorized")
|
||||
return
|
||||
}
|
||||
if _, ok := handler.uploadPipelines[pipelineID]; !ok {
|
||||
writeHTTPError(w, http.StatusNotFound, "upload pipeline not found")
|
||||
return
|
||||
}
|
||||
if _, ok := token.AllowedPipelines[pipelineID]; !ok {
|
||||
writeHTTPError(w, http.StatusForbidden, "forbidden")
|
||||
return
|
||||
}
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
if err := ingest.ValidateContentType(contentType); err != nil {
|
||||
writeHTTPError(w, http.StatusUnsupportedMediaType, "unsupported content type")
|
||||
@@ -107,6 +150,7 @@ func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
record, err := handler.coordinator.Submit(r.Context(), UploadRequest{
|
||||
TokenID: token.ID,
|
||||
PipelineID: pipelineID,
|
||||
ContentType: contentType,
|
||||
Body: r.Body,
|
||||
@@ -122,6 +166,19 @@ func (handler uploadHTTPHandler) handleUpload(w http.ResponseWriter, r *http.Req
|
||||
})
|
||||
}
|
||||
|
||||
func uploadPipelineIDFromPath(path string) (string, bool) {
|
||||
const prefix = "/v1/pipelines/"
|
||||
const suffix = "/upload"
|
||||
if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) {
|
||||
return "", false
|
||||
}
|
||||
pipelineID := strings.TrimSuffix(strings.TrimPrefix(path, prefix), suffix)
|
||||
if pipelineID == "" || strings.Contains(pipelineID, "/") {
|
||||
return "", false
|
||||
}
|
||||
return pipelineID, true
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) handleRunStatus(w http.ResponseWriter, r *http.Request) {
|
||||
rawRunID := strings.TrimPrefix(r.URL.Path, "/runs/")
|
||||
if rawRunID == "" || strings.Contains(rawRunID, "/") {
|
||||
@@ -136,17 +193,17 @@ func (handler uploadHTTPHandler) handleRunStatus(w http.ResponseWriter, r *http.
|
||||
writeJSON(w, http.StatusOK, record)
|
||||
}
|
||||
|
||||
func (handler uploadHTTPHandler) authenticate(header string) (string, bool) {
|
||||
func (handler uploadHTTPHandler) authenticate(header string) (resolvedUploadToken, bool) {
|
||||
const prefix = "Bearer "
|
||||
if !strings.HasPrefix(header, prefix) {
|
||||
return "", false
|
||||
return resolvedUploadToken{}, false
|
||||
}
|
||||
token := strings.TrimSpace(strings.TrimPrefix(header, prefix))
|
||||
if token == "" {
|
||||
return "", false
|
||||
return resolvedUploadToken{}, false
|
||||
}
|
||||
pipelineID, ok := handler.tokens[token]
|
||||
return pipelineID, ok
|
||||
resolved, ok := handler.tokens[token]
|
||||
return resolved, ok
|
||||
}
|
||||
|
||||
func uploadIdempotencyKey(header http.Header) (string, error) {
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/ingest"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
clientupload "gitea.maximumdirect.net/eric/distributor/pkg/upload"
|
||||
)
|
||||
|
||||
func TestHTTPUploadPublishesTarAndGzipFanout(t *testing.T) {
|
||||
@@ -79,14 +80,15 @@ func TestHTTPUploadInvalidArchiveIsRejectedWithoutRunID(t *testing.T) {
|
||||
}}, 4, 1))
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]string{"reports-secret": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
status, body := postHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("not a tar archive"))
|
||||
if status != http.StatusBadRequest {
|
||||
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusBadRequest, body)
|
||||
t.Fatalf("POST upload status = %d, want %d; body = %s", status, http.StatusBadRequest, body)
|
||||
}
|
||||
if strings.Contains(body, "run_id") || strings.Contains(body, "reports-secret") {
|
||||
t.Fatalf("invalid archive response exposed run id or token: %s", body)
|
||||
@@ -107,7 +109,8 @@ func TestHTTPUploadIdempotencyReturnsOriginalRunForSameBundle(t *testing.T) {
|
||||
}}, 4, 1))
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]string{"reports-secret": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
@@ -134,7 +137,8 @@ func TestHTTPUploadIdempotencyReturnsConflictForDifferentBundle(t *testing.T) {
|
||||
}}, 4, 1))
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]string{"reports-secret": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
@@ -146,7 +150,7 @@ func TestHTTPUploadIdempotencyReturnsConflictForDifferentBundle(t *testing.T) {
|
||||
ID: "weather.daily.brentwood.2026-05-31",
|
||||
}))
|
||||
if status != http.StatusConflict {
|
||||
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusConflict, body)
|
||||
t.Fatalf("POST upload status = %d, want %d; body = %s", status, http.StatusConflict, body)
|
||||
}
|
||||
if strings.Contains(body, "reports-secret") {
|
||||
t.Fatalf("conflict response exposed token: %s", body)
|
||||
@@ -168,14 +172,15 @@ func TestHTTPUploadOversizedArchiveIsRejectedWithoutRunID(t *testing.T) {
|
||||
coordinator := NewUploadCoordinator(context.Background(), cfg)
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]string{"reports-secret": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
status, body := postHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{}))
|
||||
if status != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusRequestEntityTooLarge, body)
|
||||
t.Fatalf("POST upload status = %d, want %d; body = %s", status, http.StatusRequestEntityTooLarge, body)
|
||||
}
|
||||
if strings.Contains(body, "run_id") || strings.Contains(body, "reports-secret") {
|
||||
t.Fatalf("oversized response exposed run id or token: %s", body)
|
||||
@@ -211,7 +216,8 @@ func TestHTTPUploadSamePipelineRequestsSerialize(t *testing.T) {
|
||||
})
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]string{"reports-secret": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
@@ -262,16 +268,17 @@ func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) {
|
||||
})
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]string{
|
||||
"one-secret": "reports-one",
|
||||
"two-secret": "reports-two",
|
||||
tokens: map[string]resolvedUploadToken{
|
||||
"one-secret": uploadHTTPTestToken("reports-one-reporter", "one-secret", "reports-one"),
|
||||
"two-secret": uploadHTTPTestToken("reports-two-reporter", "two-secret", "reports-two"),
|
||||
},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports-one", "reports-two"}),
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
firstRunID := submitHTTPUpload(t, server, "one-secret", ingest.ContentTypeTar, []byte("first"))
|
||||
secondRunID := submitHTTPUpload(t, server, "two-secret", ingest.ContentTypeTar, []byte("second"))
|
||||
firstRunID := submitHTTPUploadToPipeline(t, server, "reports-one", "one-secret", ingest.ContentTypeTar, []byte("first"))
|
||||
secondRunID := submitHTTPUploadToPipeline(t, server, "reports-two", "two-secret", ingest.ContentTypeTar, []byte("second"))
|
||||
waitForStartedPipelines(t, started, "reports-one", "reports-two")
|
||||
waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusRunning)
|
||||
waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusRunning)
|
||||
@@ -284,6 +291,199 @@ func TestHTTPUploadDifferentPipelinesRunConcurrently(t *testing.T) {
|
||||
waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
|
||||
}
|
||||
|
||||
func TestHTTPUploadOneTokenCanUploadToMultiplePipelines(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
|
||||
{
|
||||
id: "reports-one",
|
||||
tokenEnv: "SHARED_UPLOAD_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports-one"),
|
||||
destinations: []string{firstDestination},
|
||||
},
|
||||
{
|
||||
id: "reports-two",
|
||||
tokenEnv: "SHARED_UPLOAD_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports-two"),
|
||||
destinations: []string{secondDestination},
|
||||
},
|
||||
}, 4, 1)
|
||||
cfg.UploadTokens = []config.UploadToken{{
|
||||
ID: "shared-reporter",
|
||||
TokenEnv: "SHARED_UPLOAD_TOKEN",
|
||||
AllowPipelines: []string{"reports-one", "reports-two"},
|
||||
}}
|
||||
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"SHARED_UPLOAD_TOKEN": "shared-secret",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("newUploadHTTPHandler() error = %v", err)
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
firstRunID := submitHTTPUploadToPipeline(t, server, "reports-one", "shared-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{
|
||||
ID: "reports.one.2026-06-08",
|
||||
}))
|
||||
secondRunID := submitHTTPUploadToPipeline(t, server, "reports-two", "shared-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{
|
||||
ID: "reports.two.2026-06-08",
|
||||
}))
|
||||
|
||||
first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
|
||||
second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
|
||||
if first.PipelineID != "reports-one" || second.PipelineID != "reports-two" {
|
||||
t.Fatalf("statuses pipeline = %q/%q, want reports-one/reports-two", first.PipelineID, second.PipelineID)
|
||||
}
|
||||
assertPublishedBundle(t, firstDestination)
|
||||
assertPublishedBundle(t, secondDestination)
|
||||
}
|
||||
|
||||
func TestHTTPUploadMultipleTokensCanUploadToOnePipeline(t *testing.T) {
|
||||
destination := t.TempDir()
|
||||
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{{
|
||||
id: "reports",
|
||||
tokenEnv: "FIRST_UPLOAD_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports"),
|
||||
destinations: []string{destination},
|
||||
}}, 4, 1)
|
||||
cfg.UploadTokens = []config.UploadToken{
|
||||
{ID: "first-reporter", TokenEnv: "FIRST_UPLOAD_TOKEN", AllowPipelines: []string{"reports"}},
|
||||
{ID: "second-reporter", TokenEnv: "SECOND_UPLOAD_TOKEN", AllowPipelines: []string{"reports"}},
|
||||
}
|
||||
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"FIRST_UPLOAD_TOKEN": "first-secret",
|
||||
"SECOND_UPLOAD_TOKEN": "second-secret",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("newUploadHTTPHandler() error = %v", err)
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
firstRunID := submitHTTPUpload(t, server, "first-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{}))
|
||||
secondRunID := submitHTTPUpload(t, server, "second-secret", ingest.ContentTypeTar, bundleArchive(t, false, testutil.BundleOptions{}))
|
||||
|
||||
first := waitForHTTPUploadStatus(t, server, firstRunID, UploadStatusSucceeded)
|
||||
second := waitForHTTPUploadStatus(t, server, secondRunID, UploadStatusSucceeded)
|
||||
if first.PipelineID != "reports" || second.PipelineID != "reports" {
|
||||
t.Fatalf("statuses pipeline = %q/%q, want reports/reports", first.PipelineID, second.PipelineID)
|
||||
}
|
||||
assertPublishedBundle(t, destination)
|
||||
}
|
||||
|
||||
func TestHTTPUploadRejectsDisallowedPipelineAndLegacyUploadWithoutQueueing(t *testing.T) {
|
||||
coordinator := NewUploadCoordinator(context.Background(), httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
|
||||
{
|
||||
id: "reports",
|
||||
tokenEnv: "REPORTS_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports"),
|
||||
destinations: []string{t.TempDir()},
|
||||
},
|
||||
{
|
||||
id: "private",
|
||||
tokenEnv: "PRIVATE_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "private"),
|
||||
destinations: []string{t.TempDir()},
|
||||
},
|
||||
}, 4, 1))
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: coordinator,
|
||||
tokens: map[string]resolvedUploadToken{
|
||||
"reports-secret": uploadHTTPTestToken("reports-reporter", "reports-secret", "reports"),
|
||||
},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports", "private"}),
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
status, body := postHTTPUploadToPipeline(t, server, "private", "reports-secret", ingest.ContentTypeTar, []byte("archive"))
|
||||
if status != http.StatusForbidden {
|
||||
t.Fatalf("disallowed upload status = %d, want %d; body = %s", status, http.StatusForbidden, body)
|
||||
}
|
||||
status, body = postLegacyHTTPUpload(t, server, "reports-secret", ingest.ContentTypeTar, []byte("archive"))
|
||||
if status != http.StatusNotFound {
|
||||
t.Fatalf("legacy upload status = %d, want %d; body = %s", status, http.StatusNotFound, body)
|
||||
}
|
||||
if got := coordinator.QueueDepth(); got != 0 {
|
||||
t.Fatalf("queue depth = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPUploadPublishesThroughSelectedPipeline(t *testing.T) {
|
||||
firstDestination := t.TempDir()
|
||||
secondDestination := t.TempDir()
|
||||
cfg := httpUploadIntegrationConfig(t, []httpUploadPipelineSpec{
|
||||
{
|
||||
id: "reports-one",
|
||||
tokenEnv: "SHARED_UPLOAD_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports-one"),
|
||||
destinations: []string{firstDestination},
|
||||
},
|
||||
{
|
||||
id: "reports-two",
|
||||
tokenEnv: "SHARED_UPLOAD_TOKEN",
|
||||
stagingPath: filepath.Join(t.TempDir(), "reports-two"),
|
||||
destinations: []string{secondDestination},
|
||||
},
|
||||
}, 4, 1)
|
||||
cfg.UploadTokens = []config.UploadToken{{
|
||||
ID: "shared-reporter",
|
||||
TokenEnv: "SHARED_UPLOAD_TOKEN",
|
||||
AllowPipelines: []string{"reports-one", "reports-two"},
|
||||
}}
|
||||
handler, err := newUploadHTTPHandler(context.Background(), cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"SHARED_UPLOAD_TOKEN": "shared-secret",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("newUploadHTTPHandler() error = %v", err)
|
||||
}
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
|
||||
bundleRoot := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, bundleRoot, "", testutil.BundleOptions{
|
||||
ID: "reports.selected.2026-06-08",
|
||||
})
|
||||
client, err := clientupload.NewClient(clientupload.ClientOptions{
|
||||
Endpoint: server.URL,
|
||||
Token: "shared-secret",
|
||||
HTTPClient: server.Client(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
result, err := client.UploadBundle(context.Background(), clientupload.UploadBundleOptions{
|
||||
PipelineID: "reports-two",
|
||||
Root: bundleRoot,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadBundle() error = %v", err)
|
||||
}
|
||||
runID := UploadRunID(result.RunID)
|
||||
record := waitForHTTPUploadStatus(t, server, runID, UploadStatusSucceeded)
|
||||
|
||||
if record.PipelineID != "reports-two" {
|
||||
t.Fatalf("record pipeline = %q, want reports-two", record.PipelineID)
|
||||
}
|
||||
if record.Report == nil {
|
||||
t.Fatal("completed status report = nil, want run report")
|
||||
}
|
||||
if got, want := len(record.Report.Pipelines), 1; got != want {
|
||||
t.Fatalf("report pipeline count = %d, want %d", got, want)
|
||||
}
|
||||
if record.Report.Pipelines[0].ID != "reports-two" {
|
||||
t.Fatalf("report pipeline = %q, want reports-two", record.Report.Pipelines[0].ID)
|
||||
}
|
||||
if got, want := len(record.Report.Actions), 1; got != want {
|
||||
t.Fatalf("report action count = %d, want %d", got, want)
|
||||
}
|
||||
if record.Report.Actions[0].PipelineID != "reports-two" {
|
||||
t.Fatalf("action pipeline = %q, want reports-two", record.Report.Actions[0].PipelineID)
|
||||
}
|
||||
assertDirectoryEmpty(t, firstDestination)
|
||||
assertPublishedBundle(t, secondDestination)
|
||||
}
|
||||
|
||||
type httpUploadPipelineSpec struct {
|
||||
id string
|
||||
tokenEnv string
|
||||
@@ -311,7 +511,6 @@ func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpe
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{
|
||||
TokenEnv: spec.tokenEnv,
|
||||
StagingPath: spec.stagingPath,
|
||||
MaxUploadSize: &size,
|
||||
},
|
||||
@@ -326,6 +525,11 @@ func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpe
|
||||
})
|
||||
}
|
||||
cfg.Pipelines = append(cfg.Pipelines, pipeline)
|
||||
cfg.UploadTokens = append(cfg.UploadTokens, config.UploadToken{
|
||||
ID: spec.id + "-reporter",
|
||||
TokenEnv: spec.tokenEnv,
|
||||
AllowPipelines: []string{spec.id},
|
||||
})
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
return cfg
|
||||
@@ -333,7 +537,12 @@ func httpUploadIntegrationConfig(t *testing.T, pipelines []httpUploadPipelineSpe
|
||||
|
||||
func submitHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) UploadRunID {
|
||||
t.Helper()
|
||||
status, responseBody := postHTTPUpload(t, server, token, contentType, body)
|
||||
return submitHTTPUploadToPipeline(t, server, "reports", token, contentType, body)
|
||||
}
|
||||
|
||||
func submitHTTPUploadToPipeline(t *testing.T, server *httptest.Server, pipelineID, token, contentType string, body []byte) UploadRunID {
|
||||
t.Helper()
|
||||
status, responseBody := postHTTPUploadToPipeline(t, server, pipelineID, token, contentType, body)
|
||||
return decodeAcceptedHTTPUpload(t, status, responseBody)
|
||||
}
|
||||
|
||||
@@ -346,7 +555,7 @@ func submitHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, conte
|
||||
func decodeAcceptedHTTPUpload(t *testing.T, status int, responseBody string) UploadRunID {
|
||||
t.Helper()
|
||||
if status != http.StatusAccepted {
|
||||
t.Fatalf("POST /upload status = %d, want %d; body = %s", status, http.StatusAccepted, responseBody)
|
||||
t.Fatalf("POST upload status = %d, want %d; body = %s", status, http.StatusAccepted, responseBody)
|
||||
}
|
||||
var accepted uploadAcceptedResponse
|
||||
if err := json.Unmarshal([]byte(responseBody), &accepted); err != nil {
|
||||
@@ -360,12 +569,22 @@ func decodeAcceptedHTTPUpload(t *testing.T, status int, responseBody string) Upl
|
||||
|
||||
func postHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) (int, string) {
|
||||
t.Helper()
|
||||
return postHTTPUploadWithKey(t, server, token, contentType, "", body)
|
||||
return postHTTPUploadToPipeline(t, server, "reports", token, contentType, body)
|
||||
}
|
||||
|
||||
func postHTTPUploadToPipeline(t *testing.T, server *httptest.Server, pipelineID, token, contentType string, body []byte) (int, string) {
|
||||
t.Helper()
|
||||
return postHTTPUploadWithKeyToPipeline(t, server, pipelineID, token, contentType, "", body)
|
||||
}
|
||||
|
||||
func postHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, contentType, key string, body []byte) (int, string) {
|
||||
t.Helper()
|
||||
request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
|
||||
return postHTTPUploadWithKeyToPipeline(t, server, "reports", token, contentType, key, body)
|
||||
}
|
||||
|
||||
func postHTTPUploadWithKeyToPipeline(t *testing.T, server *httptest.Server, pipelineID, token, contentType, key string, body []byte) (int, string) {
|
||||
t.Helper()
|
||||
request, err := http.NewRequest(http.MethodPost, server.URL+"/v1/pipelines/"+pipelineID+"/upload", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("NewRequest() error = %v", err)
|
||||
}
|
||||
@@ -376,7 +595,27 @@ func postHTTPUploadWithKey(t *testing.T, server *httptest.Server, token, content
|
||||
}
|
||||
response, err := server.Client().Do(request)
|
||||
if err != nil {
|
||||
t.Fatalf("POST /upload error = %v", err)
|
||||
t.Fatalf("POST upload error = %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
data, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read response body: %v", err)
|
||||
}
|
||||
return response.StatusCode, string(data)
|
||||
}
|
||||
|
||||
func postLegacyHTTPUpload(t *testing.T, server *httptest.Server, token, contentType string, body []byte) (int, string) {
|
||||
t.Helper()
|
||||
request, err := http.NewRequest(http.MethodPost, server.URL+"/upload", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("NewRequest() error = %v", err)
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
request.Header.Set("Content-Type", contentType)
|
||||
response, err := server.Client().Do(request)
|
||||
if err != nil {
|
||||
t.Fatalf("POST legacy upload error = %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
data, err := io.ReadAll(response.Body)
|
||||
|
||||
@@ -53,10 +53,14 @@ func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) {
|
||||
ID: "weekly",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{TokenEnv: "OTHER_UPLOAD_TOKEN"},
|
||||
},
|
||||
Destinations: cfg.Pipelines[0].Destinations,
|
||||
})
|
||||
cfg.UploadTokens = append(cfg.UploadTokens, config.UploadToken{
|
||||
ID: "weekly-reporter",
|
||||
TokenEnv: "OTHER_UPLOAD_TOKEN",
|
||||
AllowPipelines: []string{"weekly"},
|
||||
})
|
||||
config.ApplyDefaults(&cfg)
|
||||
secret := "super-secret-token"
|
||||
_, err = resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
@@ -71,6 +75,39 @@ func TestResolveUploadTokensFailsForMissingAndDuplicateTokens(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveUploadTokensAllowsMultiplePipelines(t *testing.T) {
|
||||
cfg := uploadHTTPTestConfig()
|
||||
cfg.Pipelines = append(cfg.Pipelines, config.Pipeline{
|
||||
ID: "weekly",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
},
|
||||
Destinations: cfg.Pipelines[0].Destinations,
|
||||
})
|
||||
cfg.UploadTokens[0].AllowPipelines = []string{"reports", "weekly"}
|
||||
config.ApplyDefaults(&cfg)
|
||||
|
||||
tokens, err := resolveUploadTokens(cfg, uploadHTTPTestEnvironment(map[string]string{
|
||||
"UPLOAD_TOKEN": "secret",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("resolveUploadTokens() error = %v", err)
|
||||
}
|
||||
token, ok := tokens["secret"]
|
||||
if !ok {
|
||||
t.Fatal("resolved token missing")
|
||||
}
|
||||
if token.ID != "reporter" || token.Value != "secret" {
|
||||
t.Fatalf("resolved token = %#v, want id and value", token)
|
||||
}
|
||||
if _, ok := token.AllowedPipelines["reports"]; !ok {
|
||||
t.Fatalf("allowed pipelines = %#v, want reports", token.AllowedPipelines)
|
||||
}
|
||||
if _, ok := token.AllowedPipelines["weekly"]; !ok {
|
||||
t.Fatalf("allowed pipelines = %#v, want weekly", token.AllowedPipelines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewUploadHTTPHandlerAcceptsDefaultedConfig(t *testing.T) {
|
||||
cfg := uploadHTTPTestConfig()
|
||||
cfg.Server.HTTP.Bind = ""
|
||||
@@ -109,10 +146,11 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
|
||||
return UploadRunRecord{ID: "reports.20260603T120000Z.abcdef12", Status: UploadStatusAccepted}, nil
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
request.Header.Set("Idempotency-Key", "producer.retry:20260603")
|
||||
@@ -125,6 +163,9 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
|
||||
if submitted.PipelineID != "reports" {
|
||||
t.Fatalf("submitted pipeline = %q, want reports", submitted.PipelineID)
|
||||
}
|
||||
if submitted.TokenID != "reporter" {
|
||||
t.Fatalf("submitted token id = %q, want reporter", submitted.TokenID)
|
||||
}
|
||||
if submitted.IdempotencyKey != "producer.retry:20260603" {
|
||||
t.Fatalf("submitted idempotency key = %q, want producer.retry:20260603", submitted.IdempotencyKey)
|
||||
}
|
||||
@@ -143,12 +184,13 @@ func TestUploadHTTPHandlerAuthenticatesAndAcceptsUpload(t *testing.T) {
|
||||
func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
|
||||
for _, authHeader := range []string{"", "Bearer wrong-token"} {
|
||||
for _, authHeader := range []string{"", "Basic valid-token", "Bearer", "Bearer wrong-token"} {
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", authHeader)
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
|
||||
@@ -163,7 +205,73 @@ func TestUploadHTTPHandlerRejectsUnauthorizedRequests(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t *testing.T) {
|
||||
func TestUploadHTTPHandlerRejectsForbiddenPipeline(t *testing.T) {
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{
|
||||
submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
|
||||
t.Fatal("Submit should not be called")
|
||||
return UploadRunRecord{}, nil
|
||||
},
|
||||
},
|
||||
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports", "private"}),
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/private/upload", strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, http.StatusForbidden, recorder.Body.String())
|
||||
}
|
||||
if strings.Contains(recorder.Body.String(), "valid-token") {
|
||||
t.Fatalf("forbidden response exposed token: %q", recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRejectsInvalidPathAndRemovedLegacyUpload(t *testing.T) {
|
||||
handler := uploadHTTPHandler{
|
||||
coordinator: fakeUploadCoordinator{
|
||||
submit: func(context.Context, UploadRequest) (UploadRunRecord, error) {
|
||||
t.Fatal("Submit should not be called")
|
||||
return UploadRunRecord{}, nil
|
||||
},
|
||||
},
|
||||
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
wantStatus int
|
||||
}{
|
||||
{name: "legacy upload", url: "/upload", wantStatus: http.StatusNotFound},
|
||||
{name: "missing pipeline", url: "/v1/pipelines//upload", wantStatus: http.StatusNotFound},
|
||||
{name: "extra segment", url: "/v1/pipelines/reports/upload/extra", wantStatus: http.StatusNotFound},
|
||||
{name: "invalid pipeline id", url: "/v1/pipelines/.reports/upload", wantStatus: http.StatusBadRequest},
|
||||
{name: "pipeline query", url: "/v1/pipelines/reports/upload?pipeline=other", wantStatus: http.StatusBadRequest},
|
||||
{name: "pipeline id query", url: "/v1/pipelines/reports/upload?pipeline_id=other", wantStatus: http.StatusBadRequest},
|
||||
{name: "unknown upload pipeline", url: "/v1/pipelines/missing/upload", wantStatus: http.StatusNotFound},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, tt.url, strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != tt.wantStatus {
|
||||
t.Fatalf("status = %d, want %d; body = %q", recorder.Code, tt.wantStatus, recorder.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadHTTPHandlerRejectsUnsupportedContentTypeAndInvalidKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
@@ -174,14 +282,14 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
|
||||
}{
|
||||
{
|
||||
name: "unsupported content type",
|
||||
url: "/upload",
|
||||
url: "/v1/pipelines/reports/upload",
|
||||
contentType: "application/zip",
|
||||
body: strings.NewReader("archive"),
|
||||
wantStatus: http.StatusUnsupportedMediaType,
|
||||
},
|
||||
{
|
||||
name: "invalid key syntax",
|
||||
url: "/upload",
|
||||
url: "/v1/pipelines/reports/upload",
|
||||
contentType: "application/x-tar",
|
||||
keyValues: []string{"bad key"},
|
||||
body: strings.NewReader("archive"),
|
||||
@@ -189,7 +297,7 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
|
||||
},
|
||||
{
|
||||
name: "empty key",
|
||||
url: "/upload",
|
||||
url: "/v1/pipelines/reports/upload",
|
||||
contentType: "application/x-tar",
|
||||
keyValues: []string{""},
|
||||
body: strings.NewReader("archive"),
|
||||
@@ -197,7 +305,7 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
|
||||
},
|
||||
{
|
||||
name: "too long key",
|
||||
url: "/upload",
|
||||
url: "/v1/pipelines/reports/upload",
|
||||
contentType: "application/x-tar",
|
||||
keyValues: []string{strings.Repeat("a", 129)},
|
||||
body: strings.NewReader("archive"),
|
||||
@@ -205,19 +313,12 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
|
||||
},
|
||||
{
|
||||
name: "multiple keys",
|
||||
url: "/upload",
|
||||
url: "/v1/pipelines/reports/upload",
|
||||
contentType: "application/x-tar",
|
||||
keyValues: []string{"one", "two"},
|
||||
body: strings.NewReader("archive"),
|
||||
wantStatus: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "submitted pipeline id",
|
||||
url: "/upload?pipeline_id=reports",
|
||||
contentType: "application/x-tar",
|
||||
body: strings.NewReader("archive"),
|
||||
wantStatus: http.StatusBadRequest,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -228,7 +329,8 @@ func TestUploadHTTPHandlerRejectsUnsupportedContentTypeInvalidKeyAndPipelineID(t
|
||||
return UploadRunRecord{}, nil
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, tt.url, tt.body)
|
||||
@@ -270,10 +372,11 @@ func TestUploadHTTPHandlerMapsSubmitErrors(t *testing.T) {
|
||||
return UploadRunRecord{}, tt.err
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPost, "/upload", strings.NewReader("archive"))
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/pipelines/reports/upload", strings.NewReader("archive"))
|
||||
request.Header.Set("Authorization", "Bearer valid-token")
|
||||
request.Header.Set("Content-Type", "application/x-tar")
|
||||
|
||||
@@ -306,7 +409,8 @@ func TestUploadHTTPHandlerRunStatusAndHealth(t *testing.T) {
|
||||
}, true
|
||||
},
|
||||
},
|
||||
tokens: map[string]string{"valid-token": "reports"},
|
||||
tokens: map[string]resolvedUploadToken{"valid-token": uploadHTTPTestToken("reporter", "valid-token", "reports")},
|
||||
uploadPipelines: pipelineIDSet([]string{"reports"}),
|
||||
}
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
@@ -362,7 +466,6 @@ func uploadHTTPTestConfig() config.Config {
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendHTTPUpload,
|
||||
Upload: config.HTTPUpload{
|
||||
TokenEnv: "UPLOAD_TOKEN",
|
||||
StagingPath: "/tmp/distributor-test/reports",
|
||||
MaxUploadSize: &size,
|
||||
},
|
||||
@@ -374,6 +477,11 @@ func uploadHTTPTestConfig() config.Config {
|
||||
Publish: &config.PublishPolicy{Source: true},
|
||||
}},
|
||||
}},
|
||||
UploadTokens: []config.UploadToken{{
|
||||
ID: "reporter",
|
||||
TokenEnv: "UPLOAD_TOKEN",
|
||||
AllowPipelines: []string{"reports"},
|
||||
}},
|
||||
}
|
||||
config.ApplyDefaults(&cfg)
|
||||
return cfg
|
||||
@@ -384,3 +492,11 @@ func uploadHTTPTestEnvironment(values map[string]string) config.Environment {
|
||||
return "", false
|
||||
})
|
||||
}
|
||||
|
||||
func uploadHTTPTestToken(id, value string, pipelines ...string) resolvedUploadToken {
|
||||
return resolvedUploadToken{
|
||||
ID: id,
|
||||
Value: value,
|
||||
AllowedPipelines: pipelineIDSet(pipelines),
|
||||
}
|
||||
}
|
||||
|
||||
88
internal/cli/prune.go
Normal file
88
internal/cli/prune.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||
)
|
||||
|
||||
func pruneCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
if hasHelp(args) {
|
||||
printPruneHelp(stdout)
|
||||
return exitOK
|
||||
}
|
||||
|
||||
flags := newFlagSet("prune", stderr)
|
||||
configPath := flags.String("config", "", "path to config file")
|
||||
pipelineID := flags.String("pipeline", "", "pipeline id")
|
||||
destinationID := flags.String("destination", "", "destination id")
|
||||
dryRun := flags.Bool("dry-run", false, "report planned deletes without deleting outputs or rewriting state")
|
||||
apply := flags.Bool("apply", false, "delete planned managed outputs and rewrite state")
|
||||
formatFlag := addFormatFlag(flags)
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if rejectPositionalArgs(stderr, "prune", flags.Args()) {
|
||||
return exitUsage
|
||||
}
|
||||
format, ok := parseOutputFormat(stderr, "prune", *formatFlag)
|
||||
if !ok {
|
||||
return exitUsage
|
||||
}
|
||||
if !validatePruneFlags(stderr, *configPath, *pipelineID, *destinationID, *dryRun, *apply) {
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
if _, err := app.Prune(ctx, app.PruneOptions{
|
||||
ConfigPath: *configPath,
|
||||
PipelineID: *pipelineID,
|
||||
DestinationID: *destinationID,
|
||||
DryRun: *dryRun,
|
||||
Stdout: stdout,
|
||||
OutputFormat: format,
|
||||
}); err != nil {
|
||||
return fail(stderr, err)
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func validatePruneFlags(stderr io.Writer, configPath, pipelineID, destinationID string, dryRun, apply bool) bool {
|
||||
if configPath == "" {
|
||||
fmt.Fprintf(stderr, "%s: prune requires --config\n", app.Name)
|
||||
return false
|
||||
}
|
||||
if pipelineID == "" {
|
||||
fmt.Fprintf(stderr, "%s: prune requires --pipeline\n", app.Name)
|
||||
return false
|
||||
}
|
||||
if destinationID == "" {
|
||||
fmt.Fprintf(stderr, "%s: prune requires --destination\n", app.Name)
|
||||
return false
|
||||
}
|
||||
if dryRun == apply {
|
||||
fmt.Fprintf(stderr, "%s: prune requires exactly one of --dry-run or --apply\n", app.Name)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func printPruneHelp(w io.Writer) {
|
||||
fmt.Fprint(w, `Usage:
|
||||
distributor prune --config <path> --pipeline <id> --destination <id> (--dry-run|--apply) [--format text|json]
|
||||
|
||||
Options:
|
||||
--config <path> Path to config file
|
||||
--pipeline <id> Pipeline id that selects the destination root
|
||||
--destination <id> Destination id that selects the destination root
|
||||
--dry-run Report planned deletes without deleting outputs or rewriting state
|
||||
--apply Delete planned managed outputs and rewrite state
|
||||
--format text|json Output format
|
||||
|
||||
Prune reads the selected destination's configured retention policy and managed
|
||||
state, then plans owner-scoped managed output deletion. --dry-run is read-only.
|
||||
--apply deletes only planned managed output paths, preserves unmanaged files,
|
||||
and rewrites state after confirmed deletes.
|
||||
`)
|
||||
}
|
||||
210
internal/cli/prune_test.go
Normal file
210
internal/cli/prune_test.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestExecutePruneRejectsInvalidFlags(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
wantStderr string
|
||||
}{
|
||||
{
|
||||
name: "missing config",
|
||||
args: []string{"prune", "--pipeline", "reports", "--destination", "archive", "--dry-run"},
|
||||
wantStderr: "requires --config",
|
||||
},
|
||||
{
|
||||
name: "missing pipeline",
|
||||
args: []string{"prune", "--config", "config.yml", "--destination", "archive", "--dry-run"},
|
||||
wantStderr: "requires --pipeline",
|
||||
},
|
||||
{
|
||||
name: "missing destination",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--dry-run"},
|
||||
wantStderr: "requires --destination",
|
||||
},
|
||||
{
|
||||
name: "missing mode",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive"},
|
||||
wantStderr: "requires exactly one of --dry-run or --apply",
|
||||
},
|
||||
{
|
||||
name: "conflicting modes",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "--apply"},
|
||||
wantStderr: "requires exactly one of --dry-run or --apply",
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "--format", "xml"},
|
||||
wantStderr: "format must be text or json",
|
||||
},
|
||||
{
|
||||
name: "positional",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "extra"},
|
||||
wantStderr: "does not accept positional arguments",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := Execute(context.Background(), tt.args, &stdout, &stderr)
|
||||
if code != exitUsage {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitUsage, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tt.wantStderr) {
|
||||
t.Fatalf("stderr = %q, want substring %q", stderr.String(), tt.wantStderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePruneDryRunReportsWithoutWriting(t *testing.T) {
|
||||
destinationRoot, configPath := writePruneLocalFixture(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"prune",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
"--dry-run",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if output := stdout.String(); !strings.Contains(output, "status=would_change") || !strings.Contains(output, "planned=2") || !strings.Contains(output, "deleted=0") {
|
||||
t.Fatalf("stdout = %q, want dry-run prune summary", output)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePruneJSONReport(t *testing.T) {
|
||||
_, configPath := writePruneLocalFixture(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"prune",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
"--dry-run",
|
||||
"--format", "json",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
envelope := decodeEnvelope(t, &stdout)
|
||||
if envelope["command"] != "prune" || envelope["ok"] != true {
|
||||
t.Fatalf("envelope = %#v, want prune ok", envelope)
|
||||
}
|
||||
result := envelopeResult(t, envelope)
|
||||
if result["would_change"] != true || result["state_changed"] != false || result["dry_run"] != true {
|
||||
t.Fatalf("result = %#v, want dry-run pending change", result)
|
||||
}
|
||||
planned, ok := result["planned_outputs"].([]any)
|
||||
if !ok || len(planned) != 2 {
|
||||
t.Fatalf("planned outputs = %#v, want two", result["planned_outputs"])
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePruneApplyDeletesManagedOutputs(t *testing.T) {
|
||||
destinationRoot, configPath := writePruneLocalFixture(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"prune",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
"--apply",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if output := stdout.String(); !strings.Contains(output, "status=changed") || !strings.Contains(output, "deleted=2") {
|
||||
t.Fatalf("stdout = %q, want applied prune summary", output)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) {
|
||||
t.Fatalf("report.md stat error = %v, want not exist", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "summary.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("summary.txt stat error = %v, want not exist", err)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
|
||||
t.Fatalf("state file stat error = %v", err)
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := state.ManagedOutputPaths(destinationState); len(got) != 0 {
|
||||
t.Fatalf("state outputs = %#v, want none", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func writePruneLocalFixture(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
|
||||
for _, file := range testutil.DefaultSourceFiles() {
|
||||
path := filepath.Join(destinationRoot, filepath.FromSlash(file.Path))
|
||||
if err := os.WriteFile(path, []byte(file.Data), 0o600); err != nil {
|
||||
t.Fatalf("write destination output: %v", err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged output: %v", err)
|
||||
}
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
config := `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: ` + sourceRoot + `
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: ` + destinationRoot + `
|
||||
retention:
|
||||
prune:
|
||||
enabled: true
|
||||
older_than: 1h
|
||||
`
|
||||
if err := os.WriteFile(configPath, []byte(strings.TrimSpace(config)+"\n"), 0o600); err != nil {
|
||||
t.Fatalf("write prune config: %v", err)
|
||||
}
|
||||
return destinationRoot, configPath
|
||||
}
|
||||
84
internal/cli/reconcile_state.go
Normal file
84
internal/cli/reconcile_state.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||
)
|
||||
|
||||
func reconcileStateCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
if hasHelp(args) {
|
||||
printReconcileStateHelp(stdout)
|
||||
return exitOK
|
||||
}
|
||||
|
||||
flags := newFlagSet("reconcile-state", stderr)
|
||||
configPath := flags.String("config", "", "path to config file")
|
||||
pipelineID := flags.String("pipeline", "", "pipeline id")
|
||||
destinationID := flags.String("destination", "", "destination id")
|
||||
allOwners := flags.Bool("all-owners", false, "repair all shared-root owners in the selected destination root")
|
||||
dryRun := flags.Bool("dry-run", false, "report repairs without rewriting state")
|
||||
formatFlag := addFormatFlag(flags)
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if rejectPositionalArgs(stderr, "reconcile-state", flags.Args()) {
|
||||
return exitUsage
|
||||
}
|
||||
format, ok := parseOutputFormat(stderr, "reconcile-state", *formatFlag)
|
||||
if !ok {
|
||||
return exitUsage
|
||||
}
|
||||
if !validateReconcileStateFlags(stderr, *configPath, *pipelineID, *destinationID) {
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
if _, err := app.ReconcileState(ctx, app.ReconcileStateOptions{
|
||||
ConfigPath: *configPath,
|
||||
PipelineID: *pipelineID,
|
||||
DestinationID: *destinationID,
|
||||
AllOwners: *allOwners,
|
||||
DryRun: *dryRun,
|
||||
Stdout: stdout,
|
||||
OutputFormat: format,
|
||||
}); err != nil {
|
||||
return fail(stderr, err)
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func validateReconcileStateFlags(stderr io.Writer, configPath, pipelineID, destinationID string) bool {
|
||||
if configPath == "" {
|
||||
fmt.Fprintf(stderr, "%s: reconcile-state requires --config\n", app.Name)
|
||||
return false
|
||||
}
|
||||
if pipelineID == "" {
|
||||
fmt.Fprintf(stderr, "%s: reconcile-state requires --pipeline\n", app.Name)
|
||||
return false
|
||||
}
|
||||
if destinationID == "" {
|
||||
fmt.Fprintf(stderr, "%s: reconcile-state requires --destination\n", app.Name)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func printReconcileStateHelp(w io.Writer) {
|
||||
fmt.Fprint(w, `Usage:
|
||||
distributor reconcile-state --config <path> --pipeline <id> --destination <id> [--all-owners] [--dry-run] [--format text|json]
|
||||
|
||||
Options:
|
||||
--config <path> Path to config file
|
||||
--pipeline <id> Pipeline id that selects the destination root
|
||||
--destination <id> Destination id that selects the destination root
|
||||
--all-owners Repair all shared-root owners in the selected destination root
|
||||
--dry-run Report repairs without rewriting state
|
||||
--format text|json Output format
|
||||
|
||||
Reconcile-state checks managed output records against destination storage and
|
||||
removes records for missing managed outputs unless --dry-run is set. It reports
|
||||
unmanaged entries but does not delete or adopt destination files.
|
||||
`)
|
||||
}
|
||||
190
internal/cli/reconcile_state_test.go
Normal file
190
internal/cli/reconcile_state_test.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestExecuteReconcileStateAppliesByDefault(t *testing.T) {
|
||||
_, destinationRoot, configPath := writeReconcileStateLocalFixture(t)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nSunny.\n"), 0o600); err != nil {
|
||||
t.Fatalf("write managed output: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged output: %v", err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"reconcile-state",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "status=changed") {
|
||||
t.Fatalf("stdout = %q, want changed status", stdout.String())
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
|
||||
t.Fatalf("state outputs = %q, want report.md", got)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReconcileStateDryRunReportsWithoutWriting(t *testing.T) {
|
||||
_, destinationRoot, configPath := writeReconcileStateLocalFixture(t)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nSunny.\n"), 0o600); err != nil {
|
||||
t.Fatalf("write managed output: %v", err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"reconcile-state",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
"--dry-run",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "status=would_change") {
|
||||
t.Fatalf("stdout = %q, want would_change status", stdout.String())
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReconcileStateJSONReport(t *testing.T) {
|
||||
_, destinationRoot, configPath := writeReconcileStateLocalFixture(t)
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("# Report\nSunny.\n"), 0o600); err != nil {
|
||||
t.Fatalf("write managed output: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged output: %v", err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"reconcile-state",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
"--dry-run",
|
||||
"--format", "json",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
envelope := decodeEnvelope(t, &stdout)
|
||||
if envelope["command"] != "reconcile-state" || envelope["ok"] != true {
|
||||
t.Fatalf("envelope = %#v, want reconcile-state ok", envelope)
|
||||
}
|
||||
result := envelopeResult(t, envelope)
|
||||
if result["would_change"] != true || result["changed"] != false || result["dry_run"] != true {
|
||||
t.Fatalf("result = %#v, want dry-run pending change", result)
|
||||
}
|
||||
missing, ok := result["missing_managed_outputs"].([]any)
|
||||
if !ok || len(missing) != 1 {
|
||||
t.Fatalf("missing outputs = %#v, want one", result["missing_managed_outputs"])
|
||||
}
|
||||
unmanaged, ok := result["unmanaged_entries"].([]any)
|
||||
if !ok || len(unmanaged) != 1 {
|
||||
t.Fatalf("unmanaged entries = %#v, want one", result["unmanaged_entries"])
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReconcileStateRejectsInvalidFlags(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
wantStderr string
|
||||
}{
|
||||
{
|
||||
name: "missing config",
|
||||
args: []string{"reconcile-state", "--pipeline", "reports", "--destination", "archive"},
|
||||
wantStderr: "requires --config",
|
||||
},
|
||||
{
|
||||
name: "missing pipeline",
|
||||
args: []string{"reconcile-state", "--config", "config.yml", "--destination", "archive"},
|
||||
wantStderr: "requires --pipeline",
|
||||
},
|
||||
{
|
||||
name: "missing destination",
|
||||
args: []string{"reconcile-state", "--config", "config.yml", "--pipeline", "reports"},
|
||||
wantStderr: "requires --destination",
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
args: []string{"reconcile-state", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--format", "xml"},
|
||||
wantStderr: "format must be text or json",
|
||||
},
|
||||
{
|
||||
name: "positional",
|
||||
args: []string{"reconcile-state", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "extra"},
|
||||
wantStderr: "does not accept positional arguments",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := Execute(context.Background(), tt.args, &stdout, &stderr)
|
||||
if code != exitUsage {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitUsage, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tt.wantStderr) {
|
||||
t.Fatalf("stderr = %q, want substring %q", stderr.String(), tt.wantStderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeReconcileStateLocalFixture(t *testing.T) (string, string, string) {
|
||||
t.Helper()
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
|
||||
configPath := testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot)
|
||||
return sourceRoot, destinationRoot, configPath
|
||||
}
|
||||
|
||||
func assertLocalFile(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read file %s: %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("file %s = %q, want %q", path, data, want)
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,10 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
return versionCommand(ctx, args[1:], stdout, stderr)
|
||||
case "run":
|
||||
return runCommand(ctx, args[1:], stdout, stderr)
|
||||
case "reconcile-state":
|
||||
return reconcileStateCommand(ctx, args[1:], stdout, stderr)
|
||||
case "prune":
|
||||
return pruneCommand(ctx, args[1:], stdout, stderr)
|
||||
case "serve":
|
||||
return serveCommand(ctx, args[1:], stdout, stderr)
|
||||
case "validate":
|
||||
@@ -53,6 +57,9 @@ Usage:
|
||||
Commands:
|
||||
version Print version information
|
||||
run Run configured distribution pipelines
|
||||
reconcile-state
|
||||
Repair missing managed-output records in destination state
|
||||
prune Prune managed outputs using configured retention policy
|
||||
serve Run the HTTP upload server
|
||||
validate Validate a source bundle or bundle tree
|
||||
inspect Inspect bundles or distributor state
|
||||
|
||||
@@ -53,12 +53,15 @@ func TestBackendViewValidationKeepsHTTPUploadSourceOnly(t *testing.T) {
|
||||
ID: "reports",
|
||||
Source: Backend{
|
||||
Backend: BackendHTTPUpload,
|
||||
Upload: HTTPUpload{TokenEnv: "UPLOAD_TOKEN"},
|
||||
},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendHTTPUpload,
|
||||
}},
|
||||
}}, UploadTokens: []UploadToken{{
|
||||
ID: "reporter",
|
||||
TokenEnv: "UPLOAD_TOKEN",
|
||||
AllowPipelines: []string{"reports"},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
type Config struct {
|
||||
Server Server `yaml:"server"`
|
||||
Secrets Secrets `yaml:"secrets"`
|
||||
UploadTokens []UploadToken `yaml:"upload_tokens"`
|
||||
Pipelines []Pipeline `yaml:"pipelines"`
|
||||
}
|
||||
|
||||
@@ -23,6 +24,12 @@ type Secrets struct {
|
||||
Directory string `yaml:"directory"`
|
||||
}
|
||||
|
||||
type UploadToken struct {
|
||||
ID string `yaml:"id"`
|
||||
TokenEnv string `yaml:"token_env"`
|
||||
AllowPipelines []string `yaml:"allow_pipelines"`
|
||||
}
|
||||
|
||||
type Pipeline struct {
|
||||
ID string `yaml:"id"`
|
||||
Source Backend `yaml:"source"`
|
||||
@@ -48,6 +55,9 @@ type Destination struct {
|
||||
Transform Transform `yaml:"transform"`
|
||||
PathMap PathMapping `yaml:"path_mapping"`
|
||||
Links *Links `yaml:"links"`
|
||||
State StatePolicy `yaml:"state"`
|
||||
Reconciliation ReconciliationPolicy `yaml:"reconciliation"`
|
||||
Retention RetentionPolicy `yaml:"retention"`
|
||||
Transfer TransferPolicy `yaml:"transfer"`
|
||||
}
|
||||
|
||||
@@ -68,7 +78,6 @@ type Backend struct {
|
||||
}
|
||||
|
||||
type HTTPUpload struct {
|
||||
TokenEnv string `yaml:"token_env"`
|
||||
StagingPath string `yaml:"staging_path"`
|
||||
MaxUploadSize *ByteSize `yaml:"max_upload_size"`
|
||||
}
|
||||
@@ -112,6 +121,24 @@ type Links struct {
|
||||
Primary string `yaml:"primary"`
|
||||
}
|
||||
|
||||
type ReconciliationPolicy struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type StatePolicy struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type RetentionPolicy struct {
|
||||
Prune PrunePolicy `yaml:"prune"`
|
||||
}
|
||||
|
||||
type PrunePolicy struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
OlderThan *Duration `yaml:"older_than"`
|
||||
KeepLatest *int `yaml:"keep_latest"`
|
||||
}
|
||||
|
||||
type TransferPolicy struct {
|
||||
OnDestinationSame string `yaml:"on_destination_same"`
|
||||
OnDestinationOlder string `yaml:"on_destination_older"`
|
||||
|
||||
@@ -42,6 +42,16 @@ const (
|
||||
LinkPrimarySource = "source"
|
||||
)
|
||||
|
||||
const (
|
||||
ReconciliationModeReplace = "replace"
|
||||
ReconciliationModeMerge = "merge"
|
||||
)
|
||||
|
||||
const (
|
||||
StateModeSingleOwner = "single_owner"
|
||||
StateModeSharedRoot = "shared_root"
|
||||
)
|
||||
|
||||
const DefaultS3Region = "us-east-1"
|
||||
|
||||
const (
|
||||
@@ -79,6 +89,12 @@ func ApplyDefaults(cfg *Config) {
|
||||
if destination.Links != nil && destination.Links.Primary == "" {
|
||||
destination.Links.Primary = LinkPrimaryAuto
|
||||
}
|
||||
if destination.State.Mode == "" {
|
||||
destination.State.Mode = StateModeSingleOwner
|
||||
}
|
||||
if destination.Reconciliation.Mode == "" {
|
||||
destination.Reconciliation.Mode = ReconciliationModeReplace
|
||||
}
|
||||
if destination.Transfer.OnDestinationSame == "" {
|
||||
destination.Transfer.OnDestinationSame = TransferActionSkip
|
||||
}
|
||||
|
||||
@@ -33,6 +33,15 @@ pipelines:
|
||||
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
|
||||
t.Fatalf("transfer default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.Reconciliation.Mode, ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode default = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destination.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode default = %q, want %q", got, want)
|
||||
}
|
||||
if destination.Retention.Prune.Enabled {
|
||||
t.Fatal("retention.prune.enabled default = true, want false")
|
||||
}
|
||||
if cfg.Secrets.Directory != "" {
|
||||
t.Fatalf("secrets.directory = %q, want empty", cfg.Secrets.Directory)
|
||||
}
|
||||
@@ -164,6 +173,94 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsExplicitReconciliationModes(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
reconciliation:
|
||||
mode: replace
|
||||
- id: web
|
||||
backend: local
|
||||
path: /web
|
||||
reconciliation:
|
||||
mode: merge
|
||||
`)
|
||||
|
||||
destinations := cfg.Pipelines[0].Destinations
|
||||
if got, want := destinations[0].Reconciliation.Mode, ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("archive reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destinations[1].Reconciliation.Mode, ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("web reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsExplicitStateModes(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
state:
|
||||
mode: single_owner
|
||||
- id: web
|
||||
backend: local
|
||||
path: /web
|
||||
state:
|
||||
mode: shared_root
|
||||
`)
|
||||
|
||||
destinations := cfg.Pipelines[0].Destinations
|
||||
if got, want := destinations[0].State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("archive state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := destinations[1].State.Mode, StateModeSharedRoot; got != want {
|
||||
t.Fatalf("web state mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsRetentionPruneConfig(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: /source
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
retention:
|
||||
prune:
|
||||
enabled: true
|
||||
older_than: 168h
|
||||
keep_latest: 3
|
||||
`)
|
||||
|
||||
prune := cfg.Pipelines[0].Destinations[0].Retention.Prune
|
||||
if !prune.Enabled {
|
||||
t.Fatal("retention.prune.enabled = false, want true")
|
||||
}
|
||||
if prune.OlderThan == nil || prune.OlderThan.String() != "168h0m0s" {
|
||||
t.Fatalf("retention.prune.older_than = %v, want 168h", prune.OlderThan)
|
||||
}
|
||||
if prune.KeepLatest == nil || *prune.KeepLatest != 3 {
|
||||
t.Fatalf("retention.prune.keep_latest = %v, want 3", prune.KeepLatest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsFixedPathMapping(t *testing.T) {
|
||||
cfg := loadConfig(t, `
|
||||
pipelines:
|
||||
@@ -256,13 +353,17 @@ pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
staging_path: /srv/distributor/staging/weather-daily
|
||||
max_upload_size: 32MB
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
upload_tokens:
|
||||
- id: weather-reporter
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
allow_pipelines:
|
||||
- weather-daily
|
||||
`)
|
||||
|
||||
server := cfg.Server.HTTP
|
||||
@@ -289,9 +390,6 @@ pipelines:
|
||||
if got, want := source.Backend, BackendHTTPUpload; got != want {
|
||||
t.Fatalf("source.backend = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.Upload.TokenEnv, "WEATHER_DAILY_UPLOAD_TOKEN"; got != want {
|
||||
t.Fatalf("source.token_env = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.Upload.StagingPath, "/srv/distributor/staging/weather-daily"; got != want {
|
||||
t.Fatalf("source.staging_path = %q, want %q", got, want)
|
||||
}
|
||||
@@ -309,11 +407,15 @@ pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
upload_tokens:
|
||||
- id: weather-reporter
|
||||
token_env: WEATHER_DAILY_UPLOAD_TOKEN
|
||||
allow_pipelines:
|
||||
- weather-daily
|
||||
`)
|
||||
|
||||
source := cfg.Pipelines[0].Source
|
||||
@@ -325,6 +427,120 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileAcceptsHTTPUploadTokens(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"valid multi pipeline token": `
|
||||
pipelines:
|
||||
- id: weather-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive/weather
|
||||
- id: calendar-daily
|
||||
source:
|
||||
backend: http_upload
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive/calendar
|
||||
upload_tokens:
|
||||
- id: reporter
|
||||
token_env: REPORTER_UPLOAD_TOKEN
|
||||
allow_pipelines:
|
||||
- weather-daily
|
||||
- calendar-daily
|
||||
`,
|
||||
"multiple tokens for one pipeline": `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: http_upload
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: /archive
|
||||
upload_tokens:
|
||||
- id: reporter-a
|
||||
token_env: REPORTER_A_UPLOAD_TOKEN
|
||||
allow_pipelines:
|
||||
- reports
|
||||
- id: reporter-b
|
||||
token_env: REPORTER_B_UPLOAD_TOKEN
|
||||
allow_pipelines:
|
||||
- reports
|
||||
`,
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
loadConfig(t, body)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileRejectsInvalidUploadTokens(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
body string
|
||||
want string
|
||||
}{
|
||||
"missing token list": {
|
||||
body: `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
want: "upload_tokens is required",
|
||||
},
|
||||
"duplicate token ids": {
|
||||
body: `upload_tokens: [{id: reporter, token_env: ONE_UPLOAD_TOKEN, allow_pipelines: [reports]}, {id: reporter, token_env: TWO_UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
want: "upload token id reporter is duplicated",
|
||||
},
|
||||
"duplicate allowlist entries": {
|
||||
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports, reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
want: "allow_pipelines contains duplicate pipeline id reports",
|
||||
},
|
||||
"unknown allowed pipeline id": {
|
||||
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [missing]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
want: "references unknown pipeline missing",
|
||||
},
|
||||
"non upload allowed pipeline id": {
|
||||
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}, {id: uploader, token_env: OTHER_UPLOAD_TOKEN, allow_pipelines: [upload]}]
|
||||
pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}, {id: upload, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive-upload}]}]`,
|
||||
want: "references non-http_upload pipeline reports",
|
||||
},
|
||||
"upload pipeline not allowed": {
|
||||
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}, {id: other, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive-other}]}]`,
|
||||
want: "http_upload pipeline other is not allowed by any upload token",
|
||||
},
|
||||
"missing token id": {
|
||||
body: `upload_tokens: [{token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
want: "upload_tokens[0].id is required",
|
||||
},
|
||||
"invalid token id": {
|
||||
body: `upload_tokens: [{id: ".reporter", token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
want: "upload_tokens[0].id must be a slug-like identifier",
|
||||
},
|
||||
"missing token env": {
|
||||
body: `upload_tokens: [{id: reporter, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
want: "upload_tokens[0].token_env is required",
|
||||
},
|
||||
"missing allowlist": {
|
||||
body: `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
want: "upload_tokens[0].allow_pipelines is required",
|
||||
},
|
||||
}
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assertLoadError(t, tt.body, tt.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileValidBackendConfigs(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"local": `
|
||||
@@ -516,15 +732,22 @@ func TestLoadFileRejectsInvalidS3Config(t *testing.T) {
|
||||
func TestLoadFileRejectsInvalidHTTPUploadConfig(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"server size": `server: {http: {max_upload_size: 20XB}}`,
|
||||
"source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"zero source size": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"source size": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload, max_upload_size: 20XB}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"zero source size": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"server duration": `server: {http: {retention: forever}}`,
|
||||
"zero server duration": `server: {http: {retention: 0s}}`,
|
||||
"missing token env": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"prune duration": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, retention: {prune: {enabled: true, older_than: forever}}}]}]`,
|
||||
"missing upload tokens": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"destination http upload": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: ingest, backend: http_upload}]}]`,
|
||||
"literal token": `pipelines: [{id: reports, source: {backend: http_upload, token: secret, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"literal token": `upload_tokens: [{id: reporter, token: secret, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"unknown server field": `server: {http: {surprise: true}}`,
|
||||
"unknown source field": `pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"legacy source token env": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload, token_env: UPLOAD_TOKEN}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
"unknown source field": `upload_tokens: [{id: reporter, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]
|
||||
pipelines: [{id: reports, source: {backend: http_upload, surprise: true}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||
}
|
||||
for name, body := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
@@ -718,6 +941,8 @@ func TestExampleConfigsLoad(t *testing.T) {
|
||||
"../../examples/local-index.yml",
|
||||
"../../examples/fan-out.yml",
|
||||
"../../examples/archive-and-latest.yml",
|
||||
"../../examples/merge-reconciliation.yml",
|
||||
"../../examples/shared-root.yml",
|
||||
"../../examples/http-upload-local.yml",
|
||||
"../../examples/ssh-destination.yml",
|
||||
"../../examples/s3-destination.yml",
|
||||
|
||||
@@ -10,6 +10,10 @@ import (
|
||||
|
||||
var idPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`)
|
||||
|
||||
func IsSlugLikeID(value string) bool {
|
||||
return idPattern.MatchString(value)
|
||||
}
|
||||
|
||||
type ValidationErrors []string
|
||||
|
||||
func (e ValidationErrors) Error() string {
|
||||
@@ -29,11 +33,12 @@ func Validate(cfg Config) error {
|
||||
}
|
||||
|
||||
pipelineIDs := make(map[string]struct{}, len(cfg.Pipelines))
|
||||
uploadPipelineIDs := make(map[string]struct{})
|
||||
for pipelineIndex, pipeline := range cfg.Pipelines {
|
||||
pipelineContext := fmt.Sprintf("pipelines[%d]", pipelineIndex)
|
||||
if pipeline.ID == "" {
|
||||
errs = append(errs, pipelineContext+".id is required")
|
||||
} else if !idPattern.MatchString(pipeline.ID) {
|
||||
} else if !IsSlugLikeID(pipeline.ID) {
|
||||
errs = append(errs, pipelineContext+".id must be a slug-like identifier")
|
||||
} else if _, exists := pipelineIDs[pipeline.ID]; exists {
|
||||
errs = append(errs, "pipeline id "+pipeline.ID+" is duplicated")
|
||||
@@ -42,6 +47,9 @@ func Validate(cfg Config) error {
|
||||
}
|
||||
|
||||
errs = validateSourceBackend(errs, pipelineContext+".source", pipeline.Source)
|
||||
if pipeline.Source.Backend == BackendHTTPUpload && pipeline.ID != "" {
|
||||
uploadPipelineIDs[pipeline.ID] = struct{}{}
|
||||
}
|
||||
errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation)
|
||||
if len(pipeline.Destinations) == 0 {
|
||||
errs = append(errs, pipelineContext+".destinations is required")
|
||||
@@ -52,7 +60,7 @@ func Validate(cfg Config) error {
|
||||
destinationContext := fmt.Sprintf("%s.destinations[%d]", pipelineContext, destinationIndex)
|
||||
if destination.ID == "" {
|
||||
errs = append(errs, destinationContext+".id is required")
|
||||
} else if !idPattern.MatchString(destination.ID) {
|
||||
} else if !IsSlugLikeID(destination.ID) {
|
||||
errs = append(errs, destinationContext+".id must be a slug-like identifier")
|
||||
} else if _, exists := destinationIDs[destination.ID]; exists {
|
||||
errs = append(errs, "destination id "+destination.ID+" is duplicated in pipeline "+pipeline.ID)
|
||||
@@ -64,10 +72,15 @@ func Validate(cfg Config) error {
|
||||
errs = validatePublishTransformPolicy(errs, destinationContext, destination.Publish, destination.Transform)
|
||||
errs = validatePathMapping(errs, destinationContext+".path_mapping", destination.PathMap)
|
||||
errs = validateLinks(errs, destinationContext+".links", destination.Links)
|
||||
errs = validateStatePolicy(errs, destinationContext+".state", destination.State)
|
||||
errs = validateReconciliationPolicy(errs, destinationContext+".reconciliation", destination.Reconciliation)
|
||||
errs = validateRetentionPolicy(errs, destinationContext+".retention", destination.Retention)
|
||||
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
|
||||
}
|
||||
}
|
||||
|
||||
errs = validateUploadTokens(errs, cfg.UploadTokens, pipelineIDs, uploadPipelineIDs)
|
||||
|
||||
if len(errs) > 0 {
|
||||
return errs
|
||||
}
|
||||
@@ -112,9 +125,6 @@ func validateDestinationBackend(errs ValidationErrors, context string, destinati
|
||||
}
|
||||
|
||||
func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTPUpload) ValidationErrors {
|
||||
if upload.TokenEnv == "" {
|
||||
errs = append(errs, context+".token_env is required for http_upload backend")
|
||||
}
|
||||
if upload.StagingPath == "" {
|
||||
errs = append(errs, context+".staging_path is required for http_upload backend")
|
||||
}
|
||||
@@ -124,6 +134,70 @@ func validateHTTPUploadSource(errs ValidationErrors, context string, upload HTTP
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateUploadTokens(errs ValidationErrors, tokens []UploadToken, pipelineIDs, uploadPipelineIDs map[string]struct{}) ValidationErrors {
|
||||
if len(uploadPipelineIDs) == 0 {
|
||||
if len(tokens) > 0 {
|
||||
errs = append(errs, "upload_tokens must reference configured http_upload pipelines")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
if len(tokens) == 0 {
|
||||
return append(errs, "upload_tokens is required when any pipeline source backend is http_upload")
|
||||
}
|
||||
|
||||
tokenIDs := make(map[string]struct{}, len(tokens))
|
||||
allowedUploadPipelineIDs := make(map[string]struct{}, len(uploadPipelineIDs))
|
||||
for tokenIndex, token := range tokens {
|
||||
context := fmt.Sprintf("upload_tokens[%d]", tokenIndex)
|
||||
if token.ID == "" {
|
||||
errs = append(errs, context+".id is required")
|
||||
} else if !IsSlugLikeID(token.ID) {
|
||||
errs = append(errs, context+".id must be a slug-like identifier")
|
||||
} else if _, exists := tokenIDs[token.ID]; exists {
|
||||
errs = append(errs, "upload token id "+token.ID+" is duplicated")
|
||||
} else {
|
||||
tokenIDs[token.ID] = struct{}{}
|
||||
}
|
||||
|
||||
if token.TokenEnv == "" {
|
||||
errs = append(errs, context+".token_env is required")
|
||||
}
|
||||
if len(token.AllowPipelines) == 0 {
|
||||
errs = append(errs, context+".allow_pipelines is required")
|
||||
}
|
||||
|
||||
seenAllowed := make(map[string]struct{}, len(token.AllowPipelines))
|
||||
for allowIndex, pipelineID := range token.AllowPipelines {
|
||||
allowContext := fmt.Sprintf("%s.allow_pipelines[%d]", context, allowIndex)
|
||||
if pipelineID == "" {
|
||||
errs = append(errs, allowContext+" is required")
|
||||
continue
|
||||
}
|
||||
if _, exists := seenAllowed[pipelineID]; exists {
|
||||
errs = append(errs, context+".allow_pipelines contains duplicate pipeline id "+pipelineID)
|
||||
continue
|
||||
}
|
||||
seenAllowed[pipelineID] = struct{}{}
|
||||
if _, exists := pipelineIDs[pipelineID]; !exists {
|
||||
errs = append(errs, allowContext+" references unknown pipeline "+pipelineID)
|
||||
continue
|
||||
}
|
||||
if _, exists := uploadPipelineIDs[pipelineID]; !exists {
|
||||
errs = append(errs, allowContext+" references non-http_upload pipeline "+pipelineID)
|
||||
continue
|
||||
}
|
||||
allowedUploadPipelineIDs[pipelineID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
for pipelineID := range uploadPipelineIDs {
|
||||
if _, exists := allowedUploadPipelineIDs[pipelineID]; !exists {
|
||||
errs = append(errs, "http_upload pipeline "+pipelineID+" is not allowed by any upload token")
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateBackend(errs ValidationErrors, context string, backend backendView) ValidationErrors {
|
||||
switch backend.Backend {
|
||||
case "":
|
||||
@@ -243,6 +317,37 @@ func validateLinks(errs ValidationErrors, context string, links *Links) Validati
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateReconciliationPolicy(errs ValidationErrors, context string, policy ReconciliationPolicy) ValidationErrors {
|
||||
if policy.Mode != ReconciliationModeReplace && policy.Mode != ReconciliationModeMerge {
|
||||
errs = append(errs, context+".mode must be "+ReconciliationModeReplace+" or "+ReconciliationModeMerge)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateRetentionPolicy(errs ValidationErrors, context string, policy RetentionPolicy) ValidationErrors {
|
||||
prune := policy.Prune
|
||||
if !prune.Enabled {
|
||||
return errs
|
||||
}
|
||||
if prune.OlderThan == nil && prune.KeepLatest == nil {
|
||||
errs = append(errs, context+".prune must set older_than or keep_latest when enabled is true")
|
||||
}
|
||||
if prune.OlderThan != nil && *prune.OlderThan <= 0 {
|
||||
errs = append(errs, context+".prune.older_than must be greater than zero")
|
||||
}
|
||||
if prune.KeepLatest != nil && *prune.KeepLatest < 0 {
|
||||
errs = append(errs, context+".prune.keep_latest must be zero or greater")
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateStatePolicy(errs ValidationErrors, context string, policy StatePolicy) ValidationErrors {
|
||||
if policy.Mode != StateModeSingleOwner && policy.Mode != StateModeSharedRoot {
|
||||
errs = append(errs, context+".mode must be "+StateModeSingleOwner+" or "+StateModeSharedRoot)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
|
||||
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
|
||||
errs = append(errs, context+".on_destination_same must be skip or fail")
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidatePublishTransformPolicy(t *testing.T) {
|
||||
@@ -107,6 +108,118 @@ func TestValidatePathMapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReconciliationPolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "replace", mode: ReconciliationModeReplace},
|
||||
{name: "merge", mode: ReconciliationModeMerge},
|
||||
{name: "invalid", mode: "append", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Reconciliation: ReconciliationPolicy{Mode: tt.mode},
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
err := Validate(cfg)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateStatePolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "single owner", mode: StateModeSingleOwner},
|
||||
{name: "shared root", mode: StateModeSharedRoot},
|
||||
{name: "invalid", mode: "shared", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
State: StatePolicy{Mode: tt.mode},
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
err := Validate(cfg)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRetentionPolicy(t *testing.T) {
|
||||
olderThan := Duration(24 * time.Hour)
|
||||
zeroDuration := Duration(0)
|
||||
keepZero := 0
|
||||
keepThree := 3
|
||||
keepNegative := -1
|
||||
tests := []struct {
|
||||
name string
|
||||
retention RetentionPolicy
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "disabled"},
|
||||
{name: "older than", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, OlderThan: &olderThan}}},
|
||||
{name: "keep zero", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, KeepLatest: &keepZero}}},
|
||||
{name: "keep latest", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, KeepLatest: &keepThree}}},
|
||||
{name: "combined", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, OlderThan: &olderThan, KeepLatest: &keepThree}}},
|
||||
{name: "missing policy", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true}}, wantErr: true},
|
||||
{name: "zero older than", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, OlderThan: &zeroDuration}}, wantErr: true},
|
||||
{name: "negative keep latest", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, KeepLatest: &keepNegative}}, wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Config{Pipelines: []Pipeline{{
|
||||
ID: "reports",
|
||||
Source: Backend{Backend: BackendLocal, Path: "/source"},
|
||||
Destinations: []Destination{{
|
||||
ID: "archive",
|
||||
Backend: BackendLocal,
|
||||
Path: "/destination",
|
||||
Retention: tt.retention,
|
||||
}},
|
||||
}}}
|
||||
ApplyDefaults(&cfg)
|
||||
err := Validate(cfg)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLinks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -6,15 +6,20 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
||||
switch plan.Action {
|
||||
case ActionSkipSame, ActionSkipDestinationNewer:
|
||||
return nil
|
||||
case ActionPublishNew, ActionReplaceOlder, ActionForceReplace:
|
||||
if usesSharedRootState(req, plan) {
|
||||
return executeSharedRoot(ctx, req, plan)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
||||
}
|
||||
@@ -23,13 +28,15 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
if plan.ExistingState == nil {
|
||||
return fmt.Errorf("replace requires existing destination state")
|
||||
}
|
||||
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, stateOutputManagedPaths(plan.ExistingState.Outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
if plan.Reconciliation.Mode == config.ReconciliationModeReplace {
|
||||
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, state.ManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if plan.Action == ActionForceReplace {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
@@ -38,10 +45,20 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if usesMergeRetention(plan) {
|
||||
if err := ensureMergeOutputPaths(ctx, req.DestinationBackend, req.DestinationBundlePath, plan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
cleanup := func() {
|
||||
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(writtenOutputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||
outputs := writtenOutputs
|
||||
if usesMergeRetention(plan) {
|
||||
outputs = newOutputs
|
||||
}
|
||||
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||
}
|
||||
for _, output := range plan.Outputs {
|
||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
||||
@@ -62,21 +79,39 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: false, PreferAtomic: true}); err != nil {
|
||||
managed := outputManagedByExistingState(output, plan.ExistingState)
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed && usesMergeRetention(plan), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
writtenOutputs = append(writtenOutputs, output)
|
||||
if !managed {
|
||||
newOutputs = append(newOutputs, output)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
createdAt := now
|
||||
if plan.ExistingState != nil {
|
||||
createdAt = plan.ExistingState.CreatedAt
|
||||
}
|
||||
stateOutputs, err := stateOutputsForPlan(plan, now)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
destinationState := state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
DistributorVersion: req.DistributorVersion,
|
||||
PipelineID: req.PipelineID,
|
||||
DestinationID: req.DestinationID,
|
||||
PublishedAt: time.Now().UTC(),
|
||||
PublishedAt: now,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
||||
Outputs: StateOutputFiles(plan.Outputs),
|
||||
Outputs: stateOutputs,
|
||||
}
|
||||
if plan.PrimaryURL != "" {
|
||||
destinationState.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
||||
@@ -96,9 +131,248 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: false, PreferAtomic: true}); err != nil {
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: plan.ExistingState != nil, PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func executeSharedRoot(ctx context.Context, req Request, plan Plan) error {
|
||||
plan.Reconciliation = normalizeReconciliation(plan.Reconciliation)
|
||||
if plan.Action == ActionForceReplace {
|
||||
if err := req.DestinationBackend.DeletePrefix(ctx, req.DestinationBundlePath, storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if plan.Action == ActionReplaceOlder && plan.Reconciliation.Mode == config.ReconciliationModeReplace {
|
||||
if err := req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, sharedRootOutputPaths(plan.OwnerOutputsToDelete), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
newOutputs := make([]Output, 0, len(plan.Outputs))
|
||||
cleanup := func() {
|
||||
outputs := writtenOutputs
|
||||
if plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
outputs = newOutputs
|
||||
}
|
||||
_ = req.DestinationBackend.DeleteManagedOutputs(ctx, req.DestinationBundlePath, ManagedOutputPaths(outputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||
}
|
||||
for _, output := range plan.Outputs {
|
||||
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data := output.Data
|
||||
if output.Kind == state.OutputKindSource {
|
||||
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
}
|
||||
managed := outputManagedBySharedRootPlan(output, plan)
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: managed, PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
writtenOutputs = append(writtenOutputs, output)
|
||||
if !managed {
|
||||
newOutputs = append(newOutputs, output)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
sharedRootState, err := sharedRootStateForPlan(req, plan, now)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if err := state.ValidateSharedRoot(sharedRootState); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(sharedRootState, "", " ")
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: sharedRootStateWriteOverwrites(plan), PreferAtomic: true}); err != nil {
|
||||
cleanup()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureMergeOutputPaths(ctx context.Context, backend storage.Backend, bundlePath string, plan Plan) error {
|
||||
for _, output := range plan.Outputs {
|
||||
if outputManagedByExistingState(output, plan.ExistingState) {
|
||||
continue
|
||||
}
|
||||
destinationPath, err := storage.Join(bundlePath, output.DestinationPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return fmt.Errorf("merge output path %s exists but is not managed by destination state", storage.DisplayPath(output.DestinationPath))
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func usesSharedRootState(req Request, plan Plan) bool {
|
||||
return normalizeState(req.State).Mode == config.StateModeSharedRoot || plan.StateMode == config.StateModeSharedRoot
|
||||
}
|
||||
|
||||
func outputManagedByExistingState(output Output, existing *state.DistributorState) bool {
|
||||
if existing == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := state.FindOutputByPath(existing.Outputs, output.DestinationPath)
|
||||
return ok
|
||||
}
|
||||
|
||||
func outputManagedBySharedRootPlan(output Output, plan Plan) bool {
|
||||
for _, existing := range currentOwnerSharedRootOutputs(plan) {
|
||||
if existing.Path == output.DestinationPath {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func stateOutputsForPlan(plan Plan, now time.Time) ([]state.OutputFile, error) {
|
||||
existingOutputs := []state.OutputFile(nil)
|
||||
if plan.ExistingState != nil {
|
||||
existingOutputs = plan.ExistingState.Outputs
|
||||
}
|
||||
planned := state.ProjectOutputs(StateOutputProjections(plan.Outputs), existingOutputs, now)
|
||||
if !usesMergeRetention(plan) || plan.ExistingState == nil {
|
||||
return planned, nil
|
||||
}
|
||||
return state.MergeOutputFiles(plan.ExistingState.Outputs, planned)
|
||||
}
|
||||
|
||||
func usesMergeRetention(plan Plan) bool {
|
||||
return plan.Reconciliation.Mode == config.ReconciliationModeMerge && plan.Action == ActionReplaceOlder
|
||||
}
|
||||
|
||||
func sharedRootStateForPlan(req Request, plan Plan, now time.Time) (state.SharedRootState, error) {
|
||||
now = now.UTC()
|
||||
scope := plan.OwnerScope
|
||||
if scope.PipelineID == "" && scope.DestinationID == "" {
|
||||
scope = state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
}
|
||||
base := sharedRootBaseState(req, plan, now)
|
||||
owner := state.OwnerRecord{
|
||||
Scope: scope,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: plan.Reconciliation.Mode},
|
||||
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
||||
}
|
||||
if plan.PrimaryURL != "" {
|
||||
owner.Links = &state.LinkState{PrimaryURL: plan.PrimaryURL}
|
||||
}
|
||||
planned := state.ProjectSharedRootOutputs(StateOutputProjections(plan.Outputs), currentOwnerSharedRootOutputs(plan), scope, req.SourceBundle.Manifest, now)
|
||||
if plan.Action == ActionReplaceOlder && plan.Reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
return state.MergeOwnerOutputs(base, scope, owner, planned)
|
||||
}
|
||||
return state.ReplaceOwnerOutputs(base, scope, owner, planned)
|
||||
}
|
||||
|
||||
func sharedRootBaseState(req Request, plan Plan, now time.Time) state.SharedRootState {
|
||||
if plan.Action == ActionForceReplace {
|
||||
return newSharedRootState(req, now)
|
||||
}
|
||||
if plan.ExistingSharedRoot != nil {
|
||||
base := *plan.ExistingSharedRoot
|
||||
base.Owners = append([]state.OwnerRecord(nil), plan.ExistingSharedRoot.Owners...)
|
||||
base.Outputs = append([]state.SharedRootOutputFile(nil), plan.ExistingSharedRoot.Outputs...)
|
||||
base.DistributorVersion = req.DistributorVersion
|
||||
base.UpdatedAt = now
|
||||
return base
|
||||
}
|
||||
if plan.ExistingState != nil {
|
||||
base := newSharedRootState(req, now)
|
||||
base.CreatedAt = plan.ExistingState.CreatedAt
|
||||
base.UpdatedAt = now
|
||||
return base
|
||||
}
|
||||
return newSharedRootState(req, now)
|
||||
}
|
||||
|
||||
func newSharedRootState(req Request, now time.Time) state.SharedRootState {
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: req.DistributorVersion,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{},
|
||||
Outputs: []state.SharedRootOutputFile{},
|
||||
}
|
||||
}
|
||||
|
||||
func currentOwnerSharedRootOutputs(plan Plan) []state.SharedRootOutputFile {
|
||||
if plan.ExistingSharedRoot != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingSharedRoot.Outputs))
|
||||
for _, output := range plan.ExistingSharedRoot.Outputs {
|
||||
if output.Owner == plan.OwnerScope {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
if plan.ExistingState != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(plan.ExistingState.Outputs))
|
||||
for _, output := range plan.ExistingState.Outputs {
|
||||
outputs = append(outputs, state.SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: plan.OwnerScope,
|
||||
SourceID: plan.ExistingState.Source.Manifest.ID,
|
||||
SourceDigest: plan.ExistingState.Source.Manifest.Digest,
|
||||
SourceCreated: plan.ExistingState.Source.Manifest.Created,
|
||||
CreatedAt: output.CreatedAt,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sharedRootOutputPaths(outputs []state.SharedRootOutputFile) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func sharedRootStateWriteOverwrites(plan Plan) bool {
|
||||
return plan.ExistingSharedRoot != nil || plan.ExistingState != nil || plan.Action == ActionForceReplace
|
||||
}
|
||||
|
||||
@@ -5,11 +5,15 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||
)
|
||||
|
||||
func TestExecuteCleansUpAfterWriteFailure(t *testing.T) {
|
||||
@@ -44,6 +48,250 @@ func TestExecuteCleansUpAfterWriteFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReplaceDeletesOmittedManagedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "summary.txt")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteMergeRetainsOmittedAndOverwritesManagedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), testutil.ValidManifest(testutil.BundleOptions{}).Files...)
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
existingState := testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
if got, want := len(outputs), 2; got != want {
|
||||
t.Fatalf("state output count = %d, want %d", got, want)
|
||||
}
|
||||
if !outputs["summary.txt"].CreatedAt.Equal(existingState.Outputs[1].CreatedAt) || !outputs["summary.txt"].UpdatedAt.Equal(existingState.Outputs[1].UpdatedAt) {
|
||||
t.Fatalf("retained output timestamps = %#v, want existing %#v", outputs["summary.txt"], existingState.Outputs[1])
|
||||
}
|
||||
if !outputs["report.md"].CreatedAt.Equal(existingState.Outputs[0].CreatedAt) || !outputs["report.md"].UpdatedAt.After(existingState.Outputs[0].UpdatedAt) {
|
||||
t.Fatalf("updated output timestamps = %#v, want preserved created_at and newer updated_at", outputs["report.md"])
|
||||
}
|
||||
if got, want := destinationState.Reconciliation.Mode, config.ReconciliationModeMerge; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteMergeFailsOnUnmanagedDestinationPathCollision(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = []bundle.ManifestFile{{
|
||||
Path: "summary.txt",
|
||||
SHA256: bundle.FileDigest([]byte("Summary\n")),
|
||||
Size: int64(len("Summary\n")),
|
||||
}}
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
testutil.WriteFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want unmanaged path collision")
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "unmanaged")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "summary.txt", "old")
|
||||
}
|
||||
|
||||
func TestExecuteMergeFailureCleansUpOnlyNewOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{
|
||||
{Path: "report.md", Data: "# Report\nNew.\n"},
|
||||
{Path: "new.md", Data: "new\n"},
|
||||
{Path: "fail.md", Data: "fail\n"},
|
||||
},
|
||||
})
|
||||
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "fail.md"}
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = []bundle.ManifestFile{sourceBundle.Manifest.Files[0]}
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend.Backend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
err = Execute(context.Background(), req, plan)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() error = nil, want injected failure")
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend.Backend, "report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend.Backend, "new.md")
|
||||
testutil.AssertFakeMissing(t, destinationBackend.Backend, "fail.md")
|
||||
}
|
||||
|
||||
func TestExecuteFixedPathSupportsReconciliationModes(t *testing.T) {
|
||||
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "new", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.ID = "older.source"
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
|
||||
req.DestinationBundlePath = ""
|
||||
req.PathMapping = config.PathMappingFixed
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "report.md", "# Report\nNew.\n")
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
if got, want := destinationState.Reconciliation.Mode, mode; got != want {
|
||||
t.Fatalf("state reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteReconciliationModesHonorPublishPolicies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
publish config.PublishPolicy
|
||||
transform config.Transform
|
||||
transformer TransformerResolver
|
||||
wantPaths []string
|
||||
}{
|
||||
{
|
||||
name: "source only",
|
||||
publish: config.PublishPolicy{Source: true},
|
||||
wantPaths: []string{"report.md"},
|
||||
},
|
||||
{
|
||||
name: "generated only",
|
||||
publish: config.PublishPolicy{HTML: true},
|
||||
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
|
||||
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
|
||||
Path: "report.html",
|
||||
SourcePath: "report.md",
|
||||
Transform: transform.MarkdownToHTML,
|
||||
Data: []byte("<h1>Report</h1>\n"),
|
||||
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
|
||||
Size: int64(len("<h1>Report</h1>\n")),
|
||||
}}}},
|
||||
wantPaths: []string{"report.html"},
|
||||
},
|
||||
{
|
||||
name: "source and generated",
|
||||
publish: config.PublishPolicy{Source: true, HTML: true},
|
||||
transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
|
||||
transformer: testResolver{transform.MarkdownToHTML: testTransformer{outputs: []transform.Output{{
|
||||
Path: "report.html",
|
||||
SourcePath: "report.md",
|
||||
Transform: transform.MarkdownToHTML,
|
||||
Data: []byte("<h1>Report</h1>\n"),
|
||||
SHA256: bundle.FileDigest([]byte("<h1>Report</h1>\n")),
|
||||
Size: int64(len("<h1>Report</h1>\n")),
|
||||
}}}},
|
||||
wantPaths: []string{"report.md", "report.html"},
|
||||
},
|
||||
}
|
||||
for _, mode := range []string{config.ReconciliationModeReplace, config.ReconciliationModeMerge} {
|
||||
for _, tt := range tests {
|
||||
t.Run(mode+" "+tt.name, func(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
older := sourceBundle.Manifest
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
testutil.WriteFakeDestinationState(t, destinationBackend, "", older, testutil.DestinationStateOptions{})
|
||||
|
||||
req := testRequest(sourceBackend, destinationBackend, sourceBundle, mode)
|
||||
req.Publish = tt.publish
|
||||
req.Transform = tt.transform
|
||||
req.Transformers = tt.transformer
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
destinationState := readFakeState(t, destinationBackend, "")
|
||||
outputs := outputsByPath(destinationState.Outputs)
|
||||
for _, path := range tt.wantPaths {
|
||||
if _, ok := outputs[path]; !ok {
|
||||
t.Fatalf("state outputs = %#v, missing %s", destinationState.Outputs, path)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type failingBackend struct {
|
||||
*fake.Backend
|
||||
failPath string
|
||||
@@ -62,3 +310,48 @@ func (b *failingBackend) WriteFrom(ctx context.Context, path string, r io.Reader
|
||||
}
|
||||
return b.Backend.WriteFrom(ctx, path, r, opts)
|
||||
}
|
||||
|
||||
func testRequest(sourceBackend storage.Backend, destinationBackend storage.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
|
||||
Transfer: config.TransferPolicy{
|
||||
OnDestinationSame: config.TransferActionSkip,
|
||||
OnDestinationOlder: config.TransferActionReplace,
|
||||
OnDestinationNewer: config.TransferActionSkip,
|
||||
OnConflict: config.TransferActionFail,
|
||||
},
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeState(t *testing.T, backend storage.Backend, bundlePath string) state.DistributorState {
|
||||
t.Helper()
|
||||
statePath, err := storage.StatePath(bundlePath)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
data, err := backend.ReadFile(context.Background(), statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func outputsByPath(outputs []state.OutputFile) map[string]state.OutputFile {
|
||||
byPath := make(map[string]state.OutputFile, len(outputs))
|
||||
for _, output := range outputs {
|
||||
byPath[output.Path] = output
|
||||
}
|
||||
return byPath
|
||||
}
|
||||
|
||||
@@ -109,6 +109,18 @@ func (o Output) StateOutputFile() state.OutputFile {
|
||||
}
|
||||
}
|
||||
|
||||
func (o Output) StateOutputProjection() state.OutputProjection {
|
||||
return state.OutputProjection{
|
||||
Path: o.DestinationPath,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
}
|
||||
}
|
||||
|
||||
func (o Output) ManagedPath() string {
|
||||
return o.DestinationPath
|
||||
}
|
||||
@@ -121,6 +133,14 @@ func StateOutputFiles(outputs []Output) []state.OutputFile {
|
||||
return files
|
||||
}
|
||||
|
||||
func StateOutputProjections(outputs []Output) []state.OutputProjection {
|
||||
projections := make([]state.OutputProjection, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
projections = append(projections, output.StateOutputProjection())
|
||||
}
|
||||
return projections
|
||||
}
|
||||
|
||||
func ManagedOutputPaths(outputs []Output) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
@@ -128,11 +148,3 @@ func ManagedOutputPaths(outputs []Output) []string {
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func stateOutputManagedPaths(outputs []state.OutputFile) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
@@ -34,6 +34,8 @@ type Request struct {
|
||||
Publish config.PublishPolicy
|
||||
Transform config.Transform
|
||||
Links *config.Links
|
||||
State config.StatePolicy
|
||||
Reconciliation config.ReconciliationPolicy
|
||||
Transformers TransformerResolver
|
||||
Transfer config.TransferPolicy
|
||||
DistributorVersion string
|
||||
@@ -55,8 +57,16 @@ type Plan struct {
|
||||
Reason string
|
||||
Force bool
|
||||
PrimaryURL string
|
||||
StateMode string
|
||||
OwnerScope state.OwnerScope
|
||||
Reconciliation config.ReconciliationPolicy
|
||||
Outputs []Output
|
||||
ExistingState *state.DistributorState
|
||||
ExistingSharedRoot *state.SharedRootState
|
||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
||||
RetainedOwnerOutputs []state.SharedRootOutputFile
|
||||
OwnerOutputsToDelete []state.SharedRootOutputFile
|
||||
OwnerOutputsToWrite []Output
|
||||
}
|
||||
|
||||
type Output struct {
|
||||
@@ -88,6 +98,8 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
}
|
||||
comparison := compareDestination(req, status)
|
||||
action, reason := actionForComparison(comparison, req.Transfer, req.Force)
|
||||
reconciliation := normalizeReconciliation(req.Reconciliation)
|
||||
stateMode := normalizeState(req.State).Mode
|
||||
plan := Plan{
|
||||
PipelineID: req.PipelineID,
|
||||
DestinationID: req.DestinationID,
|
||||
@@ -99,8 +111,24 @@ func Build(ctx context.Context, req Request) (Plan, error) {
|
||||
Reason: reason,
|
||||
Force: action == ActionForceReplace,
|
||||
PrimaryURL: primaryURL,
|
||||
StateMode: stateMode,
|
||||
OwnerScope: state.CurrentOwnerScope(req.PipelineID, req.DestinationID),
|
||||
Reconciliation: reconciliation,
|
||||
Outputs: outputs,
|
||||
ExistingState: status.State,
|
||||
ExistingSharedRoot: status.SharedRoot,
|
||||
}
|
||||
if stateMode == config.StateModeSharedRoot {
|
||||
sharedDetails, err := planSharedRootOwner(ctx, req, status, action, reconciliation, outputs)
|
||||
plan.OtherOwnerOutputs = sharedDetails.OtherOwnerOutputs
|
||||
plan.RetainedOwnerOutputs = sharedDetails.RetainedOwnerOutputs
|
||||
plan.OwnerOutputsToDelete = sharedDetails.OwnerOutputsToDelete
|
||||
plan.OwnerOutputsToWrite = sharedDetails.OwnerOutputsToWrite
|
||||
if err != nil {
|
||||
plan.Action = sharedDetails.Action
|
||||
plan.Reason = sharedDetails.Reason
|
||||
return plan, err
|
||||
}
|
||||
}
|
||||
if action == ActionFailConflict || action == ActionFailUnmanaged {
|
||||
return plan, fmt.Errorf("%s: %s", action, reason)
|
||||
@@ -124,10 +152,52 @@ func validateRequest(req Request) error {
|
||||
if err := config.ValidatePublishTransformPolicy(req.Publish, req.Transform); err != nil {
|
||||
return fmt.Errorf("publish/transform policy: %w", err)
|
||||
}
|
||||
switch normalizeReconciliation(req.Reconciliation).Mode {
|
||||
case config.ReconciliationModeReplace, config.ReconciliationModeMerge:
|
||||
default:
|
||||
return fmt.Errorf("reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
switch normalizeState(req.State).Mode {
|
||||
case config.StateModeSingleOwner, config.StateModeSharedRoot:
|
||||
default:
|
||||
return fmt.Errorf("state.mode must be %s or %s", config.StateModeSingleOwner, config.StateModeSharedRoot)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeReconciliation(policy config.ReconciliationPolicy) config.ReconciliationPolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.ReconciliationModeReplace
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
func normalizeState(policy config.StatePolicy) config.StatePolicy {
|
||||
if policy.Mode == "" {
|
||||
policy.Mode = config.StateModeSingleOwner
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
func compareDestination(req Request, status state.DestinationStatus) state.Comparison {
|
||||
if normalizeState(req.State).Mode == config.StateModeSharedRoot {
|
||||
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
comparison := state.CompareSharedRootOwner(req.SourceBundle.Manifest, scope, status)
|
||||
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict {
|
||||
return comparison
|
||||
}
|
||||
destinationManifest, ok := sharedRootComparisonManifest(status, scope)
|
||||
if !ok {
|
||||
return comparison
|
||||
}
|
||||
if destinationManifest.Created.Before(req.SourceBundle.Manifest.Created) {
|
||||
return state.Comparison{Outcome: state.OutcomeDestinationOlder, Reason: "fixed destination source is older than selected source"}
|
||||
}
|
||||
if destinationManifest.Created.After(req.SourceBundle.Manifest.Created) {
|
||||
return state.Comparison{Outcome: state.OutcomeDestinationNewer, Reason: "fixed destination source is newer than selected source"}
|
||||
}
|
||||
return comparison
|
||||
}
|
||||
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
|
||||
if req.PathMapping != config.PathMappingFixed || comparison.Outcome != state.OutcomeDifferentSourceConflict || status.State == nil {
|
||||
return comparison
|
||||
@@ -142,6 +212,173 @@ func compareDestination(req Request, status state.DestinationStatus) state.Compa
|
||||
return comparison
|
||||
}
|
||||
|
||||
func sharedRootComparisonManifest(status state.DestinationStatus, scope state.OwnerScope) (bundle.Manifest, bool) {
|
||||
if status.SharedRoot != nil {
|
||||
return status.SharedRoot.SourceManifest(scope)
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
return status.State.Source.Manifest, true
|
||||
}
|
||||
return bundle.Manifest{}, false
|
||||
}
|
||||
|
||||
type sharedRootPlanDetails struct {
|
||||
Action Action
|
||||
Reason string
|
||||
OtherOwnerOutputs []state.SharedRootOutputFile
|
||||
RetainedOwnerOutputs []state.SharedRootOutputFile
|
||||
OwnerOutputsToDelete []state.SharedRootOutputFile
|
||||
OwnerOutputsToWrite []Output
|
||||
}
|
||||
|
||||
func planSharedRootOwner(ctx context.Context, req Request, status state.DestinationStatus, action Action, reconciliation config.ReconciliationPolicy, outputs []Output) (sharedRootPlanDetails, error) {
|
||||
scope := state.CurrentOwnerScope(req.PipelineID, req.DestinationID)
|
||||
details := sharedRootPlanDetails{Action: action}
|
||||
if !isWriteAction(action) {
|
||||
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
|
||||
return details, nil
|
||||
}
|
||||
|
||||
plannedPaths := outputPaths(outputs)
|
||||
if action == ActionForceReplace {
|
||||
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
|
||||
return details, nil
|
||||
}
|
||||
if conflict, ok := sharedRootPathOwnershipConflict(status, scope, plannedPaths); ok {
|
||||
reason := fmt.Sprintf("destination output path %s is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
details.Action = ActionFailConflict
|
||||
details.Reason = reason
|
||||
return details, fmt.Errorf("%s: %s", ActionFailConflict, reason)
|
||||
}
|
||||
if err := rejectSharedRootUnmanagedCollisions(ctx, req.DestinationBackend, req.DestinationBundlePath, status, scope, plannedPaths); err != nil {
|
||||
details.Action = ActionFailUnmanaged
|
||||
details.Reason = err.Error()
|
||||
return details, fmt.Errorf("%s: %s", ActionFailUnmanaged, err)
|
||||
}
|
||||
|
||||
details.OtherOwnerOutputs = otherOwnerOutputs(status, scope)
|
||||
ownerOutputs := currentOwnerOutputs(status, scope)
|
||||
planned := make(map[string]struct{}, len(plannedPaths))
|
||||
for _, path := range plannedPaths {
|
||||
planned[path] = struct{}{}
|
||||
}
|
||||
for _, output := range ownerOutputs {
|
||||
if _, exists := planned[output.Path]; exists {
|
||||
continue
|
||||
}
|
||||
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeReplace {
|
||||
details.OwnerOutputsToDelete = append(details.OwnerOutputsToDelete, output)
|
||||
continue
|
||||
}
|
||||
if action == ActionReplaceOlder && reconciliation.Mode == config.ReconciliationModeMerge {
|
||||
details.RetainedOwnerOutputs = append(details.RetainedOwnerOutputs, output)
|
||||
}
|
||||
}
|
||||
details.OwnerOutputsToWrite = append([]Output(nil), outputs...)
|
||||
return details, nil
|
||||
}
|
||||
|
||||
func isWriteAction(action Action) bool {
|
||||
switch action {
|
||||
case ActionPublishNew, ActionReplaceOlder, ActionForceReplace:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func outputPaths(outputs []Output) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.DestinationPath)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func sharedRootPathOwnershipConflict(status state.DestinationStatus, scope state.OwnerScope, paths []string) (state.PathOwnershipConflict, bool) {
|
||||
if status.SharedRoot != nil {
|
||||
return status.SharedRoot.PathOwnershipConflict(scope, paths)
|
||||
}
|
||||
return state.PathOwnershipConflict{}, false
|
||||
}
|
||||
|
||||
func rejectSharedRootUnmanagedCollisions(ctx context.Context, backend storage.Backend, bundlePath string, status state.DestinationStatus, scope state.OwnerScope, paths []string) error {
|
||||
for _, path := range paths {
|
||||
if pathManagedBySharedRootStatus(status, scope, path) {
|
||||
continue
|
||||
}
|
||||
destinationPath, err := storage.Join(bundlePath, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := backend.Stat(ctx, destinationPath); err == nil {
|
||||
return fmt.Errorf("destination output path %s exists but is not managed by destination state", storage.DisplayPath(path))
|
||||
} else if !storage.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pathManagedBySharedRootStatus(status state.DestinationStatus, scope state.OwnerScope, path string) bool {
|
||||
if status.SharedRoot != nil {
|
||||
_, exists := status.SharedRoot.OutputOwner(path)
|
||||
return exists
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
_, exists := state.FindOutputByPath(status.State.Outputs, path)
|
||||
return exists
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func otherOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
if status.SharedRoot == nil {
|
||||
return nil
|
||||
}
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
|
||||
for _, output := range status.SharedRoot.Outputs {
|
||||
if output.Owner != scope {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
|
||||
func currentOwnerOutputs(status state.DestinationStatus, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
if status.SharedRoot != nil {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.SharedRoot.Outputs))
|
||||
for _, output := range status.SharedRoot.Outputs {
|
||||
if output.Owner == scope {
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
if status.State != nil && status.State.PipelineID == scope.PipelineID && status.State.DestinationID == scope.DestinationID {
|
||||
outputs := make([]state.SharedRootOutputFile, 0, len(status.State.Outputs))
|
||||
for _, output := range status.State.Outputs {
|
||||
outputs = append(outputs, state.SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: scope,
|
||||
SourceID: status.State.Source.Manifest.ID,
|
||||
SourceDigest: status.State.Source.Manifest.Digest,
|
||||
SourceCreated: status.State.Source.Manifest.Created,
|
||||
CreatedAt: output.CreatedAt,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy, force bool) (Action, string) {
|
||||
switch comparison.Outcome {
|
||||
case state.OutcomeDestinationAbsent:
|
||||
|
||||
@@ -14,11 +14,11 @@ func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath
|
||||
}
|
||||
data, err := backend.ReadFile(ctx, statePath)
|
||||
if err == nil {
|
||||
destinationState, parseErr := state.Parse(data)
|
||||
document, parseErr := state.ParseDocument(data)
|
||||
if parseErr != nil {
|
||||
return state.DestinationStatus{StateErr: parseErr}, nil
|
||||
}
|
||||
return state.DestinationStatus{State: &destinationState, HasContents: true}, nil
|
||||
return state.DestinationStatus{State: document.SingleOwner, SharedRoot: document.SharedRoot, HasContents: true}, nil
|
||||
}
|
||||
if !storage.IsNotFound(err) {
|
||||
return state.DestinationStatus{}, err
|
||||
|
||||
399
internal/publish/shared_root_test.go
Normal file
399
internal/publish/shared_root_test.go
Normal file
@@ -0,0 +1,399 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestBuildSharedRootTreatsAbsentOwnerAsPublishable(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionPublishNew {
|
||||
t.Fatalf("plan action = %s, want publish_new", plan.Action)
|
||||
}
|
||||
if plan.OwnerScope != state.CurrentOwnerScope("reports", "archive") {
|
||||
t.Fatalf("owner scope = %#v, want reports/archive", plan.OwnerScope)
|
||||
}
|
||||
if got, want := len(plan.OtherOwnerOutputs), 1; got != want {
|
||||
t.Fatalf("other owner output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(plan.OwnerOutputsToWrite), 1; got != want {
|
||||
t.Fatalf("owner output write count = %d, want %d", got, want)
|
||||
}
|
||||
if len(plan.OwnerOutputsToDelete) != 0 || len(plan.RetainedOwnerOutputs) != 0 {
|
||||
t.Fatalf("delete=%#v retained=%#v, want none", plan.OwnerOutputsToDelete, plan.RetainedOwnerOutputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionReplaceOlder {
|
||||
t.Fatalf("plan action = %s, want replace_older", plan.Action)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.OwnerOutputsToDelete), "old.md"; got != want {
|
||||
t.Fatalf("owner outputs to delete = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.OtherOwnerOutputs), "other/report.md"; got != want {
|
||||
t.Fatalf("other owner outputs = %q, want %q", got, want)
|
||||
}
|
||||
if len(plan.RetainedOwnerOutputs) != 0 {
|
||||
t.Fatalf("retained owner outputs = %#v, want none", plan.RetainedOwnerOutputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if got, want := sharedRootOutputPathList(plan.RetainedOwnerOutputs), "old.md"; got != want {
|
||||
t.Fatalf("retained owner outputs = %q, want %q", got, want)
|
||||
}
|
||||
if len(plan.OwnerOutputsToDelete) != 0 {
|
||||
t.Fatalf("owner outputs to delete = %#v, want none", plan.OwnerOutputsToDelete)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootRejectsOtherOwnerPathConflict(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
sharedRoot := sharedRootStateWithOwners(t, sourceBundle.Manifest, false)
|
||||
sharedRoot.Outputs[0].Path = "report.md"
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRoot)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
|
||||
t.Fatalf("Build() error = %v, want fail_conflict", err)
|
||||
}
|
||||
if plan.Action != ActionFailConflict {
|
||||
t.Fatalf("plan action = %s, want fail_conflict", plan.Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSharedRootRejectsUnmanagedPathCollision(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/report.md", "unmanaged")
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
|
||||
t.Fatalf("Build() error = %v, want fail_unmanaged", err)
|
||||
}
|
||||
if plan.Action != ActionFailUnmanaged {
|
||||
t.Fatalf("plan action = %s, want fail_unmanaged", plan.Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootPublishesOwnerAndPreservesOtherOwners(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, false))
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := len(destinationState.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := destinationState.State.Mode, state.StateModeSharedRoot; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("other", "archive")); !ok {
|
||||
t.Fatal("other owner missing from shared-root state")
|
||||
}
|
||||
if _, ok := destinationState.Owner(state.CurrentOwnerScope("reports", "archive")); !ok {
|
||||
t.Fatal("current owner missing from shared-root state")
|
||||
}
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootReplaceDeletesOnlyCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
existing := sharedRootStateWithOwners(t, sourceBundle.Manifest, true)
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", existing)
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/old.md")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
if !destinationState.CreatedAt.Equal(existing.CreatedAt) {
|
||||
t.Fatalf("created_at = %s, want %s", destinationState.CreatedAt, existing.CreatedAt)
|
||||
}
|
||||
output, ok := findSharedRootOutputForTest(destinationState.Outputs, "report.md")
|
||||
if !ok {
|
||||
t.Fatal("report.md missing from shared-root outputs")
|
||||
}
|
||||
if !output.CreatedAt.Equal(existing.Outputs[1].CreatedAt) || !output.UpdatedAt.After(existing.Outputs[1].UpdatedAt) {
|
||||
t.Fatalf("report.md timestamps = created:%s updated:%s, want preserved created and newer updated", output.CreatedAt, output.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootMergeRetainsCurrentOwnerOmittedOutputs(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\nNew.\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
writeFakeSharedRootState(t, destinationBackend, "bundle", sharedRootStateWithOwners(t, sourceBundle.Manifest, true))
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeMerge)
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\nNew.\n")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/old.md", "old")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/other/report.md", "old")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := strings.Join(destinationState.AllManagedOutputPaths(), ","), "other/report.md,report.md,old.md"; got != want {
|
||||
t.Fatalf("managed paths = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSharedRootForceReplaceDeletesOnlyBundlePath(t *testing.T) {
|
||||
sourceBackend := fake.New()
|
||||
sourceBundle := testutil.WriteFakeSourceBundle(t, sourceBackend, "bundle", testutil.BundleOptions{
|
||||
Files: []testutil.SourceFile{{Path: "report.md", Data: "# Report\n"}},
|
||||
})
|
||||
destinationBackend := fake.New()
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/unmanaged.txt", "unmanaged")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle/nested/old.txt", "old")
|
||||
testutil.WriteFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
|
||||
req := sharedRootRequest(sourceBackend, destinationBackend, sourceBundle, config.ReconciliationModeReplace)
|
||||
req.Force = true
|
||||
plan, err := Build(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
if plan.Action != ActionForceReplace {
|
||||
t.Fatalf("plan action = %s, want force_replace", plan.Action)
|
||||
}
|
||||
if err := Execute(context.Background(), req, plan); err != nil {
|
||||
t.Fatalf("Execute() error = %v", err)
|
||||
}
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle/report.md", "# Report\n")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/unmanaged.txt")
|
||||
testutil.AssertFakeMissing(t, destinationBackend, "bundle/nested/old.txt")
|
||||
testutil.AssertFakeFile(t, destinationBackend, "bundle-sibling/keep.txt", "keep")
|
||||
destinationState := readFakeSharedRootState(t, destinationBackend, "bundle")
|
||||
if got, want := len(destinationState.Owners), 1; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func sharedRootRequest(sourceBackend, destinationBackend *fake.Backend, sourceBundle bundle.Bundle, reconciliationMode string) Request {
|
||||
return Request{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
SourceBundle: sourceBundle,
|
||||
SourceBackend: sourceBackend,
|
||||
DestinationBackend: destinationBackend,
|
||||
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||
Publish: config.PublishPolicy{Source: true},
|
||||
State: config.StatePolicy{Mode: config.StateModeSharedRoot},
|
||||
Reconciliation: config.ReconciliationPolicy{Mode: reconciliationMode},
|
||||
Transfer: defaultTransfer(),
|
||||
DistributorVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string, sharedRoot state.SharedRootState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(sharedRoot, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared-root state: %v", err)
|
||||
}
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, statePath, string(append(data, '\n')))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
path, err := storage.Join(relative, output.Path)
|
||||
if err != nil {
|
||||
t.Fatalf("join output path: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, path, "old")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeSharedRootState(t *testing.T, backend *fake.Backend, relative string) state.SharedRootState {
|
||||
t.Helper()
|
||||
statePath, err := storage.StatePath(relative)
|
||||
if err != nil {
|
||||
t.Fatalf("state path: %v", err)
|
||||
}
|
||||
data, err := backend.ReadFile(context.Background(), statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
}
|
||||
destinationState, err := state.ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func findSharedRootOutputForTest(outputs []state.SharedRootOutputFile, path string) (state.SharedRootOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return state.SharedRootOutputFile{}, false
|
||||
}
|
||||
|
||||
func sharedRootStateWithOwners(t *testing.T, current bundle.Manifest, includeCurrent bool) state.SharedRootState {
|
||||
t.Helper()
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
otherManifest := testutil.ValidManifest(testutil.BundleOptions{
|
||||
ID: "other.source",
|
||||
Files: []testutil.SourceFile{{Path: "other/report.md", Data: "# Other\n"}},
|
||||
})
|
||||
sharedRoot := state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: state.CurrentOwnerScope("other", "archive"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: otherManifest},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "other/report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "other/report.md",
|
||||
SHA256: otherManifest.Files[0].SHA256,
|
||||
Size: otherManifest.Files[0].Size,
|
||||
Owner: state.CurrentOwnerScope("other", "archive"),
|
||||
SourceID: otherManifest.ID,
|
||||
SourceDigest: otherManifest.Digest,
|
||||
SourceCreated: otherManifest.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
if !includeCurrent {
|
||||
return sharedRoot
|
||||
}
|
||||
older := current
|
||||
older.Created = older.Created.Add(-time.Hour)
|
||||
older.Files = append([]bundle.ManifestFile(nil), current.Files...)
|
||||
older.Files = append(older.Files, bundle.ManifestFile{
|
||||
Path: "old.md",
|
||||
SHA256: bundle.FileDigest([]byte("old\n")),
|
||||
Size: int64(len("old\n")),
|
||||
})
|
||||
older.Digest = bundle.BundleDigest(older.Files)
|
||||
scope := state.CurrentOwnerScope("reports", "archive")
|
||||
sharedRoot.Owners = append(sharedRoot.Owners, state.OwnerRecord{
|
||||
Scope: scope,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: older},
|
||||
})
|
||||
for _, file := range older.Files {
|
||||
sharedRoot.Outputs = append(sharedRoot.Outputs, state.SharedRootOutputFile{
|
||||
Path: file.Path,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: file.Path,
|
||||
SHA256: file.SHA256,
|
||||
Size: file.Size,
|
||||
Owner: scope,
|
||||
SourceID: older.ID,
|
||||
SourceDigest: older.Digest,
|
||||
SourceCreated: older.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
})
|
||||
}
|
||||
return sharedRoot
|
||||
}
|
||||
|
||||
func sharedRootOutputPathList(outputs []state.SharedRootOutputFile) string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return strings.Join(paths, ",")
|
||||
}
|
||||
@@ -22,6 +22,7 @@ const (
|
||||
|
||||
type DestinationStatus struct {
|
||||
State *DistributorState
|
||||
SharedRoot *SharedRootState
|
||||
StateErr error
|
||||
HasContents bool
|
||||
}
|
||||
@@ -31,6 +32,58 @@ type Comparison struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
func CompareSharedRootOwner(source bundle.Manifest, scope OwnerScope, status DestinationStatus) Comparison {
|
||||
if status.StateErr != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error()}
|
||||
}
|
||||
if status.SharedRoot != nil {
|
||||
if err := ValidateSharedRoot(*status.SharedRoot); err != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
|
||||
}
|
||||
owner, ok := status.SharedRoot.Owner(scope)
|
||||
if !ok {
|
||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: fmt.Sprintf("destination owner %s/%s is absent", scope.PipelineID, scope.DestinationID)}
|
||||
}
|
||||
return compareManifests(source, owner.Source.Manifest)
|
||||
}
|
||||
if status.State != nil {
|
||||
destinationState := *status.State
|
||||
if err := Validate(destinationState); err != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
|
||||
}
|
||||
if destinationState.PipelineID != scope.PipelineID {
|
||||
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationState.PipelineID, scope.PipelineID)}
|
||||
}
|
||||
if destinationState.DestinationID != scope.DestinationID {
|
||||
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("destination id %q does not match %q", destinationState.DestinationID, scope.DestinationID)}
|
||||
}
|
||||
return compareManifests(source, destinationState.Source.Manifest)
|
||||
}
|
||||
if status.HasContents {
|
||||
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state"}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
||||
}
|
||||
|
||||
func compareManifests(source, destination bundle.Manifest) Comparison {
|
||||
if manifestsEqual(source, destination) {
|
||||
return Comparison{Outcome: OutcomeSameSource, Reason: "destination source manifest matches source"}
|
||||
}
|
||||
if destination.ID != source.ID {
|
||||
return Comparison{Outcome: OutcomeDifferentSourceConflict, Reason: "destination source id differs from source"}
|
||||
}
|
||||
if destination.Created.Before(source.Created) {
|
||||
return Comparison{Outcome: OutcomeDestinationOlder, Reason: "destination source is older than source"}
|
||||
}
|
||||
if destination.Created.After(source.Created) {
|
||||
return Comparison{Outcome: OutcomeDestinationNewer, Reason: "destination source is newer than source"}
|
||||
}
|
||||
if destination.Digest != source.Digest {
|
||||
return Comparison{Outcome: OutcomeSameCreatedConflict, Reason: "destination source has same id and created time but different digest"}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: "destination source differs from source without a supported comparison outcome"}
|
||||
}
|
||||
|
||||
func Compare(source bundle.Manifest, pipelineID, destinationID string, status DestinationStatus) Comparison {
|
||||
if status.StateErr != nil {
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error()}
|
||||
@@ -53,23 +106,7 @@ func Compare(source bundle.Manifest, pipelineID, destinationID string, status De
|
||||
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("destination id %q does not match %q", destinationState.DestinationID, destinationID)}
|
||||
}
|
||||
|
||||
destinationManifest := destinationState.Source.Manifest
|
||||
if manifestsEqual(source, destinationManifest) {
|
||||
return Comparison{Outcome: OutcomeSameSource, Reason: "destination source manifest matches source"}
|
||||
}
|
||||
if destinationManifest.ID != source.ID {
|
||||
return Comparison{Outcome: OutcomeDifferentSourceConflict, Reason: "destination source id differs from source"}
|
||||
}
|
||||
if destinationManifest.Created.Before(source.Created) {
|
||||
return Comparison{Outcome: OutcomeDestinationOlder, Reason: "destination source is older than source"}
|
||||
}
|
||||
if destinationManifest.Created.After(source.Created) {
|
||||
return Comparison{Outcome: OutcomeDestinationNewer, Reason: "destination source is newer than source"}
|
||||
}
|
||||
if destinationManifest.Digest != source.Digest {
|
||||
return Comparison{Outcome: OutcomeSameCreatedConflict, Reason: "destination source has same id and created time but different digest"}
|
||||
}
|
||||
return Comparison{Outcome: OutcomeInvalidState, Reason: "destination source differs from source without a supported comparison outcome"}
|
||||
return compareManifests(source, destinationState.Source.Manifest)
|
||||
}
|
||||
|
||||
func manifestsEqual(a, b bundle.Manifest) bool {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestCompareOutcomes(t *testing.T) {
|
||||
@@ -100,11 +101,16 @@ func withState(t *testing.T, source bundle.Manifest, mutate func(*DistributorSta
|
||||
t.Helper()
|
||||
stateManifest := source
|
||||
stateManifest.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC),
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: stateManifest},
|
||||
Outputs: []OutputFile{{
|
||||
Path: "report.md",
|
||||
@@ -112,6 +118,8 @@ func withState(t *testing.T, source bundle.Manifest, mutate func(*DistributorSta
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
if mutate != nil {
|
||||
|
||||
@@ -8,9 +8,16 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
const SchemaVersion = 1
|
||||
const (
|
||||
SchemaVersion = 2
|
||||
SharedRootSchemaVersion = 3
|
||||
legacySchemaVersion = 1
|
||||
StateModeSingleOwner = config.StateModeSingleOwner
|
||||
StateModeSharedRoot = config.StateModeSharedRoot
|
||||
)
|
||||
|
||||
type DistributorState struct {
|
||||
SchemaVersion int
|
||||
@@ -18,11 +25,23 @@ type DistributorState struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
PublishedAt time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
State StatePolicy
|
||||
Reconciliation ReconciliationPolicy
|
||||
Source SourceState
|
||||
Links *LinkState
|
||||
Outputs []OutputFile
|
||||
}
|
||||
|
||||
type StatePolicy struct {
|
||||
Mode string
|
||||
}
|
||||
|
||||
type ReconciliationPolicy struct {
|
||||
Mode string
|
||||
}
|
||||
|
||||
type SourceState struct {
|
||||
Manifest bundle.Manifest
|
||||
}
|
||||
@@ -39,6 +58,8 @@ type OutputFile struct {
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type rawDistributorState struct {
|
||||
@@ -47,11 +68,23 @@ type rawDistributorState struct {
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
PublishedAt *string `json:"published_at"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
State *rawStatePolicy `json:"state"`
|
||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
||||
Source *rawSourceState `json:"source"`
|
||||
Links *rawLinkState `json:"links"`
|
||||
Outputs []rawOutputFile `json:"outputs"`
|
||||
}
|
||||
|
||||
type rawStatePolicy struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
type rawReconciliationPolicy struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
type rawSourceState struct {
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
}
|
||||
@@ -68,6 +101,8 @@ type rawOutputFile struct {
|
||||
URL string `json:"url"`
|
||||
SHA256 *string `json:"sha256"`
|
||||
Size *int64 `json:"size"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func Parse(data []byte) (DistributorState, error) {
|
||||
@@ -96,9 +131,10 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
||||
return DistributorState{}, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
state.SchemaVersion = *raw.SchemaVersion
|
||||
if state.SchemaVersion != SchemaVersion {
|
||||
return DistributorState{}, fmt.Errorf("state schema_version must be %d", SchemaVersion)
|
||||
if state.SchemaVersion != SchemaVersion && state.SchemaVersion != legacySchemaVersion {
|
||||
return DistributorState{}, fmt.Errorf("state schema_version must be %d or %d", legacySchemaVersion, SchemaVersion)
|
||||
}
|
||||
legacy := state.SchemaVersion == legacySchemaVersion
|
||||
state.DistributorVersion = raw.DistributorVersion
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return DistributorState{}, fmt.Errorf("state pipeline_id is required")
|
||||
@@ -116,6 +152,32 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
||||
return DistributorState{}, fmt.Errorf("state published_at must be RFC3339: %w", err)
|
||||
}
|
||||
state.PublishedAt = publishedAt.UTC()
|
||||
if legacy {
|
||||
state.SchemaVersion = SchemaVersion
|
||||
state.CreatedAt = state.PublishedAt
|
||||
state.UpdatedAt = state.PublishedAt
|
||||
state.State.Mode = StateModeSingleOwner
|
||||
state.Reconciliation.Mode = config.ReconciliationModeReplace
|
||||
} else {
|
||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
state.CreatedAt = createdAt
|
||||
state.UpdatedAt = updatedAt
|
||||
if raw.State == nil || raw.State.Mode == "" {
|
||||
return DistributorState{}, fmt.Errorf("state state.mode is required")
|
||||
}
|
||||
state.State.Mode = raw.State.Mode
|
||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
||||
return DistributorState{}, fmt.Errorf("state reconciliation.mode is required")
|
||||
}
|
||||
state.Reconciliation.Mode = raw.Reconciliation.Mode
|
||||
}
|
||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
||||
return DistributorState{}, fmt.Errorf("state source.manifest is required")
|
||||
}
|
||||
@@ -130,7 +192,7 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
||||
if raw.Outputs == nil {
|
||||
return DistributorState{}, fmt.Errorf("state outputs is required")
|
||||
}
|
||||
outputs, err := parseOutputs(raw.Outputs)
|
||||
outputs, err := parseOutputs(raw.Outputs, legacy, state.PublishedAt)
|
||||
if err != nil {
|
||||
return DistributorState{}, err
|
||||
}
|
||||
@@ -138,11 +200,22 @@ func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseOutputs(rawOutputs []rawOutputFile) ([]OutputFile, error) {
|
||||
func parseRequiredTime(context string, raw *string) (time.Time, error) {
|
||||
if raw == nil || *raw == "" {
|
||||
return time.Time{}, fmt.Errorf("%s is required", context)
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, *raw)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", context, err)
|
||||
}
|
||||
return parsed.UTC(), nil
|
||||
}
|
||||
|
||||
func parseOutputs(rawOutputs []rawOutputFile, legacy bool, publishedAt time.Time) ([]OutputFile, error) {
|
||||
outputs := make([]OutputFile, 0, len(rawOutputs))
|
||||
seen := make(map[string]struct{}, len(rawOutputs))
|
||||
for index, raw := range rawOutputs {
|
||||
output, err := parseOutput(index, raw)
|
||||
output, err := parseOutput(index, raw, legacy, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -155,7 +228,7 @@ func parseOutputs(rawOutputs []rawOutputFile) ([]OutputFile, error) {
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
|
||||
func parseOutput(index int, raw rawOutputFile, legacy bool, publishedAt time.Time) (OutputFile, error) {
|
||||
if raw.Path == nil || *raw.Path == "" {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||
}
|
||||
@@ -171,6 +244,19 @@ func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
|
||||
if raw.Size == nil {
|
||||
return OutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||
}
|
||||
createdAt := publishedAt
|
||||
updatedAt := publishedAt
|
||||
if !legacy {
|
||||
var err error
|
||||
createdAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
||||
if err != nil {
|
||||
return OutputFile{}, err
|
||||
}
|
||||
updatedAt, err = parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return OutputFile{}, err
|
||||
}
|
||||
}
|
||||
return OutputFile{
|
||||
Path: *raw.Path,
|
||||
Kind: *raw.Kind,
|
||||
@@ -179,6 +265,8 @@ func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
|
||||
URL: raw.URL,
|
||||
SHA256: *raw.SHA256,
|
||||
Size: *raw.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -186,6 +274,22 @@ func (s DistributorState) PublishedAtString() string {
|
||||
return s.PublishedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) CreatedAtString() string {
|
||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) UpdatedAtString() string {
|
||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o OutputFile) CreatedAtString() string {
|
||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o OutputFile) UpdatedAtString() string {
|
||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s DistributorState) MarshalJSON() ([]byte, error) {
|
||||
type sourceJSON struct {
|
||||
Manifest bundle.Manifest `json:"manifest"`
|
||||
@@ -196,6 +300,10 @@ func (s DistributorState) MarshalJSON() ([]byte, error) {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
State StatePolicy `json:"state"`
|
||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
||||
Source sourceJSON `json:"source"`
|
||||
Links *LinkState `json:"links,omitempty"`
|
||||
Outputs []OutputFile `json:"outputs"`
|
||||
@@ -206,12 +314,30 @@ func (s DistributorState) MarshalJSON() ([]byte, error) {
|
||||
PipelineID: s.PipelineID,
|
||||
DestinationID: s.DestinationID,
|
||||
PublishedAt: s.PublishedAtString(),
|
||||
CreatedAt: s.CreatedAtString(),
|
||||
UpdatedAt: s.UpdatedAtString(),
|
||||
State: s.State,
|
||||
Reconciliation: s.Reconciliation,
|
||||
Source: sourceJSON{Manifest: s.Source.Manifest},
|
||||
Links: s.Links,
|
||||
Outputs: s.Outputs,
|
||||
})
|
||||
}
|
||||
|
||||
func (p StatePolicy) MarshalJSON() ([]byte, error) {
|
||||
type policyJSON struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||
}
|
||||
|
||||
func (p ReconciliationPolicy) MarshalJSON() ([]byte, error) {
|
||||
type policyJSON struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
return json.Marshal(policyJSON{Mode: p.Mode})
|
||||
}
|
||||
|
||||
func (l LinkState) MarshalJSON() ([]byte, error) {
|
||||
type linkJSON struct {
|
||||
PrimaryURL string `json:"primary_url,omitempty"`
|
||||
@@ -228,6 +354,8 @@ func (o OutputFile) MarshalJSON() ([]byte, error) {
|
||||
URL string `json:"url,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
return json.Marshal(outputJSON{
|
||||
Path: o.Path,
|
||||
@@ -237,5 +365,7 @@ func (o OutputFile) MarshalJSON() ([]byte, error) {
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
CreatedAt: o.CreatedAt.UTC().Format(time.RFC3339),
|
||||
UpdatedAt: o.UpdatedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestParseValidState(t *testing.T) {
|
||||
@@ -24,9 +25,24 @@ func TestParseValidState(t *testing.T) {
|
||||
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("CreatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("UpdatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(state.Outputs), 1; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := state.Outputs[0].CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("output CreatedAtString() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseValidStateWithLinks(t *testing.T) {
|
||||
@@ -45,6 +61,38 @@ func TestParseValidStateWithLinks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingOutputs(t *testing.T) {
|
||||
state, err := Parse([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
state.Outputs = append(state.Outputs, OutputFile{
|
||||
Path: "summary.txt",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 10,
|
||||
CreatedAt: state.CreatedAt,
|
||||
UpdatedAt: state.UpdatedAt,
|
||||
})
|
||||
|
||||
next, changed := RemoveMissingOutputs(state, []string{"summary.txt"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingOutputs() changed = false, want true")
|
||||
}
|
||||
if got, want := ManagedOutputPaths(next), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
unchanged, changed := RemoveMissingOutputs(next, []string{"missing.txt"})
|
||||
if changed {
|
||||
t.Fatal("RemoveMissingOutputs() changed = true, want false")
|
||||
}
|
||||
if got, want := ManagedOutputPaths(unchanged), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("unchanged paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
|
||||
state, err := Parse([]byte(body))
|
||||
@@ -62,6 +110,10 @@ func TestParseRejectsMissingFields(t *testing.T) {
|
||||
"pipeline_id": `"pipeline_id"`,
|
||||
"destination_id": `"destination_id"`,
|
||||
"published_at": `"published_at"`,
|
||||
"created_at": `"created_at"`,
|
||||
"updated_at": `"updated_at"`,
|
||||
"state": `"state"`,
|
||||
"reconciliation": `"reconciliation"`,
|
||||
"source": `"source"`,
|
||||
"outputs": `"outputs"`,
|
||||
}
|
||||
@@ -75,9 +127,31 @@ func TestParseRejectsMissingFields(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidSchemaVersion(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1)
|
||||
body := strings.Replace(validStateJSON(t), `"schema_version": 2`, `"schema_version": 3`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "schema_version must be 1")
|
||||
assertStateErrorContains(t, err, "schema_version must be 1 or 2")
|
||||
}
|
||||
|
||||
func TestParseLegacyStateInfersSingleOwnerDefaults(t *testing.T) {
|
||||
state, err := Parse([]byte(legacyStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
if got, want := state.SchemaVersion, SchemaVersion; got != want {
|
||||
t.Fatalf("schema version = %d, want normalized %d", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("created_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSingleOwner; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Reconciliation.Mode, config.ReconciliationModeReplace; got != want {
|
||||
t.Fatalf("reconciliation mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.Outputs[0].UpdatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("output updated_at = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsInvalidEmbeddedManifest(t *testing.T) {
|
||||
@@ -161,6 +235,12 @@ func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
|
||||
"negative size": func(s *DistributorState) {
|
||||
s.Outputs[0].Size = -1
|
||||
},
|
||||
"missing output created_at": func(s *DistributorState) {
|
||||
s.Outputs[0].CreatedAt = time.Time{}
|
||||
},
|
||||
"missing output updated_at": func(s *DistributorState) {
|
||||
s.Outputs[0].UpdatedAt = time.Time{}
|
||||
},
|
||||
"invalid output url": func(s *DistributorState) {
|
||||
s.Outputs[0].URL = "file:///tmp/report.md"
|
||||
},
|
||||
@@ -202,13 +282,24 @@ func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
|
||||
assertStateErrorContains(t, err, "published_at must be RFC3339")
|
||||
}
|
||||
|
||||
func TestParseRejectsMalformedCreatedTimestamp(t *testing.T) {
|
||||
body := strings.Replace(validStateJSON(t), `"created_at": "2026-05-30T11:12:00Z"`, `"created_at": "May 30"`, 1)
|
||||
_, err := Parse([]byte(body))
|
||||
assertStateErrorContains(t, err, "created_at must be RFC3339")
|
||||
}
|
||||
|
||||
func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
publishedAt := time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60)),
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Outputs: []OutputFile{{
|
||||
Path: "report.md",
|
||||
@@ -216,6 +307,8 @@ func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
@@ -229,11 +322,16 @@ func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
||||
|
||||
func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
|
||||
source := validManifest(t)
|
||||
publishedAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
state := DistributorState{
|
||||
SchemaVersion: SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC),
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: StatePolicy{Mode: StateModeSingleOwner},
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
||||
Outputs: []OutputFile{{
|
||||
@@ -243,6 +341,8 @@ func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
|
||||
URL: "https://reports.example.com/archive/report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
}},
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
@@ -257,6 +357,42 @@ func TestMarshalIncludesLinksWhenPresent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputHelpers(t *testing.T) {
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
updatedAt := createdAt.Add(time.Hour)
|
||||
retained := []OutputFile{{
|
||||
Path: "old.md",
|
||||
Kind: OutputKindSource,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}}
|
||||
projected := ProjectOutputs([]OutputProjection{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
}, {
|
||||
Path: "new.md",
|
||||
Kind: OutputKindSource,
|
||||
}}, retained, updatedAt)
|
||||
if got, ok := FindOutputByPath(projected, "report.md"); !ok || !got.CreatedAt.Equal(createdAt) || !got.UpdatedAt.Equal(updatedAt) {
|
||||
t.Fatalf("projected report.md = %#v, want preserved created_at and updated updated_at", got)
|
||||
}
|
||||
merged, err := MergeOutputFiles(retained, projected)
|
||||
if err != nil {
|
||||
t.Fatalf("MergeOutputFiles() error = %v", err)
|
||||
}
|
||||
if got, want := len(merged), 3; got != want {
|
||||
t.Fatalf("merged count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := ManagedOutputPaths(DistributorState{Outputs: merged}), []string{"old.md", "report.md", "new.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("managed paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func validStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
return validStateWithManifestJSON(t, manifestJSON(t))
|
||||
@@ -265,11 +401,15 @@ func validStateJSON(t *testing.T) string {
|
||||
func validStateWithManifestJSON(t *testing.T, manifest string) string {
|
||||
t.Helper()
|
||||
return `{
|
||||
"schema_version": 1,
|
||||
"schema_version": 2,
|
||||
"distributor_version": "dev",
|
||||
"pipeline_id": "reports",
|
||||
"destination_id": "archive",
|
||||
"published_at": "2026-05-30T11:12:00Z",
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z",
|
||||
"state": {"mode": "single_owner"},
|
||||
"reconciliation": {"mode": "replace"},
|
||||
"source": {
|
||||
"manifest": ` + manifest + `
|
||||
},
|
||||
@@ -279,12 +419,31 @@ func validStateWithManifestJSON(t *testing.T, manifest string) string {
|
||||
"kind": "source",
|
||||
"source_path": "report.md",
|
||||
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
||||
"size": 16
|
||||
"size": 16,
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z"
|
||||
}
|
||||
]
|
||||
}`
|
||||
}
|
||||
|
||||
func legacyStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
return strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(strings.Replace(validStateJSON(t),
|
||||
`"schema_version": 2`, `"schema_version": 1`, 1),
|
||||
` "created_at": "2026-05-30T11:12:00Z",
|
||||
`, "", 1),
|
||||
` "updated_at": "2026-05-30T11:12:00Z",
|
||||
`, "", 1),
|
||||
` "state": {"mode": "single_owner"},
|
||||
`, "", 1),
|
||||
` "reconciliation": {"mode": "replace"},
|
||||
`, "", 1),
|
||||
`,
|
||||
"created_at": "2026-05-30T11:12:00Z",
|
||||
"updated_at": "2026-05-30T11:12:00Z"`, "", 1)
|
||||
}
|
||||
|
||||
func manifestJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("../bundle/testdata/valid_bundle/manifest.json")
|
||||
|
||||
330
internal/state/outputs.go
Normal file
330
internal/state/outputs.go
Normal file
@@ -0,0 +1,330 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
)
|
||||
|
||||
type OutputProjection struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
}
|
||||
|
||||
func FindOutputByPath(outputs []OutputFile, path string) (OutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return OutputFile{}, false
|
||||
}
|
||||
|
||||
func MergeOutputFiles(retained, planned []OutputFile) ([]OutputFile, error) {
|
||||
outputs := make([]OutputFile, 0, len(retained)+len(planned))
|
||||
indexByPath := make(map[string]int, len(retained)+len(planned))
|
||||
for _, output := range retained {
|
||||
if _, exists := indexByPath[output.Path]; exists {
|
||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
seenPlanned := make(map[string]struct{}, len(planned))
|
||||
for _, output := range planned {
|
||||
if _, exists := seenPlanned[output.Path]; exists {
|
||||
return nil, fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
seenPlanned[output.Path] = struct{}{}
|
||||
if index, exists := indexByPath[output.Path]; exists {
|
||||
outputs[index] = output
|
||||
continue
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func ManagedOutputPaths(s DistributorState) []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func RemoveMissingOutputs(s DistributorState, missingPaths []string) (DistributorState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]OutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func ProjectOutputs(outputs []OutputProjection, existing []OutputFile, now time.Time) []OutputFile {
|
||||
now = now.UTC()
|
||||
files := make([]OutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
createdAt := now
|
||||
if existingOutput, ok := FindOutputByPath(existing, output.Path); ok {
|
||||
createdAt = existingOutput.CreatedAt
|
||||
}
|
||||
files = append(files, OutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func CurrentOwnerScope(pipelineID, destinationID string) OwnerScope {
|
||||
return OwnerScope{PipelineID: pipelineID, DestinationID: destinationID}
|
||||
}
|
||||
|
||||
func (s SharedRootState) Owner(scope OwnerScope) (OwnerRecord, bool) {
|
||||
for _, owner := range s.Owners {
|
||||
if owner.Scope == scope {
|
||||
return owner, true
|
||||
}
|
||||
}
|
||||
return OwnerRecord{}, false
|
||||
}
|
||||
|
||||
func (s SharedRootState) SourceManifest(scope OwnerScope) (bundle.Manifest, bool) {
|
||||
owner, ok := s.Owner(scope)
|
||||
if !ok {
|
||||
return bundle.Manifest{}, false
|
||||
}
|
||||
return owner.Source.Manifest, true
|
||||
}
|
||||
|
||||
func (s SharedRootState) ManagedOutputPaths(scope OwnerScope) []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner == scope {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func (s SharedRootState) AllManagedOutputPaths() []string {
|
||||
paths := make([]string, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func RemoveMissingSharedRootOwnerOutputs(s SharedRootState, scope OwnerScope, missingPaths []string) (SharedRootState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner == scope {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func RemoveMissingSharedRootOutputs(s SharedRootState, missingPaths []string) (SharedRootState, bool) {
|
||||
if len(missingPaths) == 0 {
|
||||
return s, false
|
||||
}
|
||||
missing := pathSet(missingPaths)
|
||||
next := s
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs))
|
||||
changed := false
|
||||
for _, output := range s.Outputs {
|
||||
if _, remove := missing[output.Path]; remove {
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
return next, changed
|
||||
}
|
||||
|
||||
func (s SharedRootState) OutputOwner(path string) (OwnerScope, bool) {
|
||||
for _, output := range s.Outputs {
|
||||
if output.Path == path {
|
||||
return output.Owner, true
|
||||
}
|
||||
}
|
||||
return OwnerScope{}, false
|
||||
}
|
||||
|
||||
func (s SharedRootState) PathOwnershipConflict(scope OwnerScope, paths []string) (PathOwnershipConflict, bool) {
|
||||
for _, path := range paths {
|
||||
owner, exists := s.OutputOwner(path)
|
||||
if exists && owner != scope {
|
||||
return PathOwnershipConflict{Path: path, Owner: owner}, true
|
||||
}
|
||||
}
|
||||
return PathOwnershipConflict{}, false
|
||||
}
|
||||
|
||||
func ProjectSharedRootOutputs(outputs []OutputProjection, existing []SharedRootOutputFile, scope OwnerScope, source bundle.Manifest, now time.Time) []SharedRootOutputFile {
|
||||
now = now.UTC()
|
||||
files := make([]SharedRootOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
createdAt := now
|
||||
if existingOutput, ok := findSharedRootOutput(existing, output.Path); ok && existingOutput.Owner == scope {
|
||||
createdAt = existingOutput.CreatedAt
|
||||
}
|
||||
files = append(files, SharedRootOutputFile{
|
||||
Path: output.Path,
|
||||
Kind: output.Kind,
|
||||
SourcePath: output.SourcePath,
|
||||
Transform: output.Transform,
|
||||
URL: output.URL,
|
||||
SHA256: output.SHA256,
|
||||
Size: output.Size,
|
||||
Owner: scope,
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func ReplaceOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
}
|
||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
next := s
|
||||
next.Owners = upsertOwner(s.Owners, owner)
|
||||
next.Outputs = make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner != scope {
|
||||
next.Outputs = append(next.Outputs, output)
|
||||
}
|
||||
}
|
||||
next.Outputs = append(next.Outputs, planned...)
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func MergeOwnerOutputs(s SharedRootState, scope OwnerScope, owner OwnerRecord, planned []SharedRootOutputFile) (SharedRootState, error) {
|
||||
if conflict, ok := s.PathOwnershipConflict(scope, sharedRootOutputPaths(planned)); ok {
|
||||
return SharedRootState{}, fmt.Errorf("state output path %q is owned by %s/%s", conflict.Path, conflict.Owner.PipelineID, conflict.Owner.DestinationID)
|
||||
}
|
||||
if err := validatePlannedSharedRootOutputs(scope, planned); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
next := s
|
||||
next.Owners = upsertOwner(s.Owners, owner)
|
||||
outputs := make([]SharedRootOutputFile, 0, len(s.Outputs)+len(planned))
|
||||
indexByPath := make(map[string]int, len(s.Outputs)+len(planned))
|
||||
for _, output := range s.Outputs {
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
for _, output := range planned {
|
||||
if index, exists := indexByPath[output.Path]; exists {
|
||||
outputs[index] = output
|
||||
continue
|
||||
}
|
||||
indexByPath[output.Path] = len(outputs)
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
next.Outputs = outputs
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func findSharedRootOutput(outputs []SharedRootOutputFile, path string) (SharedRootOutputFile, bool) {
|
||||
for _, output := range outputs {
|
||||
if output.Path == path {
|
||||
return output, true
|
||||
}
|
||||
}
|
||||
return SharedRootOutputFile{}, false
|
||||
}
|
||||
|
||||
func sharedRootOutputPaths(outputs []SharedRootOutputFile) []string {
|
||||
paths := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
paths = append(paths, output.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func rejectDuplicateSharedRootOutputs(outputs []SharedRootOutputFile) error {
|
||||
seen := make(map[string]struct{}, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if _, exists := seen[output.Path]; exists {
|
||||
return fmt.Errorf("state output path %q is duplicated", output.Path)
|
||||
}
|
||||
seen[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePlannedSharedRootOutputs(scope OwnerScope, outputs []SharedRootOutputFile) error {
|
||||
if err := rejectDuplicateSharedRootOutputs(outputs); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, output := range outputs {
|
||||
if output.Owner != scope {
|
||||
return fmt.Errorf("state output path %q is owned by %s/%s, not %s/%s", output.Path, output.Owner.PipelineID, output.Owner.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertOwner(owners []OwnerRecord, owner OwnerRecord) []OwnerRecord {
|
||||
next := append([]OwnerRecord(nil), owners...)
|
||||
for index, existing := range next {
|
||||
if existing.Scope == owner.Scope {
|
||||
next[index] = owner
|
||||
return next
|
||||
}
|
||||
}
|
||||
return append(next, owner)
|
||||
}
|
||||
|
||||
func pathSet(paths []string) map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(paths))
|
||||
for _, path := range paths {
|
||||
set[path] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
121
internal/state/prune.go
Normal file
121
internal/state/prune.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type PruneCandidate struct {
|
||||
Path string
|
||||
UpdatedAt time.Time
|
||||
Owner *OwnerScope
|
||||
}
|
||||
|
||||
type PrunePlanOptions struct {
|
||||
Now time.Time
|
||||
OlderThan *time.Duration
|
||||
KeepLatest *int
|
||||
}
|
||||
|
||||
type PrunePlan struct {
|
||||
Pruned []PruneCandidate
|
||||
Preserved []PruneCandidate
|
||||
}
|
||||
|
||||
func SingleOwnerPruneCandidates(s DistributorState) []PruneCandidate {
|
||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
candidates = append(candidates, PruneCandidate{
|
||||
Path: output.Path,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandidate {
|
||||
candidates := make([]PruneCandidate, 0, len(s.Outputs))
|
||||
for _, output := range s.Outputs {
|
||||
if output.Owner != scope {
|
||||
continue
|
||||
}
|
||||
owner := output.Owner
|
||||
candidates = append(candidates, PruneCandidate{
|
||||
Path: output.Path,
|
||||
UpdatedAt: output.UpdatedAt,
|
||||
Owner: &owner,
|
||||
})
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func PlanPrune(candidates []PruneCandidate, options PrunePlanOptions) PrunePlan {
|
||||
ordered := append([]PruneCandidate(nil), candidates...)
|
||||
sortPruneCandidatesNewestFirst(ordered)
|
||||
if options.OlderThan == nil && options.KeepLatest == nil {
|
||||
return PrunePlan{
|
||||
Pruned: []PruneCandidate{},
|
||||
Preserved: ordered,
|
||||
}
|
||||
}
|
||||
|
||||
preservedByPath := make(map[string]struct{})
|
||||
if options.KeepLatest != nil {
|
||||
keep := *options.KeepLatest
|
||||
if keep < 0 {
|
||||
keep = 0
|
||||
}
|
||||
if keep > len(ordered) {
|
||||
keep = len(ordered)
|
||||
}
|
||||
for _, candidate := range ordered[:keep] {
|
||||
preservedByPath[candidate.Path] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
plan := PrunePlan{
|
||||
Pruned: []PruneCandidate{},
|
||||
Preserved: []PruneCandidate{},
|
||||
}
|
||||
cutoff := time.Time{}
|
||||
if options.OlderThan != nil {
|
||||
now := options.Now.UTC()
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
cutoff = now.Add(-*options.OlderThan)
|
||||
}
|
||||
|
||||
for _, candidate := range ordered {
|
||||
if _, preserved := preservedByPath[candidate.Path]; preserved {
|
||||
plan.Preserved = append(plan.Preserved, candidate)
|
||||
continue
|
||||
}
|
||||
if options.OlderThan == nil || candidate.UpdatedAt.Before(cutoff) {
|
||||
plan.Pruned = append(plan.Pruned, candidate)
|
||||
continue
|
||||
}
|
||||
plan.Preserved = append(plan.Preserved, candidate)
|
||||
}
|
||||
sortPruneCandidatesOldestFirst(plan.Pruned)
|
||||
sortPruneCandidatesNewestFirst(plan.Preserved)
|
||||
return plan
|
||||
}
|
||||
|
||||
func sortPruneCandidatesNewestFirst(candidates []PruneCandidate) {
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
if !candidates[i].UpdatedAt.Equal(candidates[j].UpdatedAt) {
|
||||
return candidates[i].UpdatedAt.After(candidates[j].UpdatedAt)
|
||||
}
|
||||
return candidates[i].Path < candidates[j].Path
|
||||
})
|
||||
}
|
||||
|
||||
func sortPruneCandidatesOldestFirst(candidates []PruneCandidate) {
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
if !candidates[i].UpdatedAt.Equal(candidates[j].UpdatedAt) {
|
||||
return candidates[i].UpdatedAt.Before(candidates[j].UpdatedAt)
|
||||
}
|
||||
return candidates[i].Path < candidates[j].Path
|
||||
})
|
||||
}
|
||||
96
internal/state/prune_test.go
Normal file
96
internal/state/prune_test.go
Normal file
@@ -0,0 +1,96 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPlanPruneOlderThan(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
olderThan := 48 * time.Hour
|
||||
plan := PlanPrune([]PruneCandidate{
|
||||
{Path: "old.txt", UpdatedAt: now.Add(-72 * time.Hour)},
|
||||
{Path: "fresh.txt", UpdatedAt: now.Add(-24 * time.Hour)},
|
||||
}, PrunePlanOptions{Now: now, OlderThan: &olderThan})
|
||||
|
||||
if got, want := pruneCandidatePaths(plan.Pruned), "old.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := pruneCandidatePaths(plan.Preserved), "fresh.txt"; got != want {
|
||||
t.Fatalf("preserved = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneKeepLatest(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
keepLatest := 2
|
||||
plan := PlanPrune([]PruneCandidate{
|
||||
{Path: "old.txt", UpdatedAt: now.Add(-72 * time.Hour)},
|
||||
{Path: "new.txt", UpdatedAt: now.Add(-1 * time.Hour)},
|
||||
{Path: "middle.txt", UpdatedAt: now.Add(-24 * time.Hour)},
|
||||
}, PrunePlanOptions{KeepLatest: &keepLatest})
|
||||
|
||||
if got, want := pruneCandidatePaths(plan.Pruned), "old.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := pruneCandidatePaths(plan.Preserved), "new.txt,middle.txt"; got != want {
|
||||
t.Fatalf("preserved = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneCombinedPolicyPreservesLatestBeforeAgeCheck(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
olderThan := 48 * time.Hour
|
||||
keepLatest := 1
|
||||
plan := PlanPrune([]PruneCandidate{
|
||||
{Path: "oldest.txt", UpdatedAt: now.Add(-96 * time.Hour)},
|
||||
{Path: "old.txt", UpdatedAt: now.Add(-72 * time.Hour)},
|
||||
{Path: "fresh.txt", UpdatedAt: now.Add(-24 * time.Hour)},
|
||||
}, PrunePlanOptions{Now: now, OlderThan: &olderThan, KeepLatest: &keepLatest})
|
||||
|
||||
if got, want := pruneCandidatePaths(plan.Pruned), "oldest.txt,old.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := pruneCandidatePaths(plan.Preserved), "fresh.txt"; got != want {
|
||||
t.Fatalf("preserved = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneDeterministicTieBreaking(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
keepLatest := 1
|
||||
plan := PlanPrune([]PruneCandidate{
|
||||
{Path: "b.txt", UpdatedAt: now},
|
||||
{Path: "a.txt", UpdatedAt: now},
|
||||
{Path: "c.txt", UpdatedAt: now.Add(-time.Hour)},
|
||||
}, PrunePlanOptions{KeepLatest: &keepLatest})
|
||||
|
||||
if got, want := pruneCandidatePaths(plan.Preserved), "a.txt"; got != want {
|
||||
t.Fatalf("preserved = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := pruneCandidatePaths(plan.Pruned), "c.txt,b.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootPruneCandidatesPreserveOtherOwners(t *testing.T) {
|
||||
sharedRoot := validSharedRootState(t)
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
candidates := SharedRootPruneCandidates(sharedRoot, scope)
|
||||
|
||||
if got, want := pruneCandidatePaths(candidates), "report.md"; got != want {
|
||||
t.Fatalf("candidates = %q, want %q", got, want)
|
||||
}
|
||||
if candidates[0].Owner == nil || *candidates[0].Owner != scope {
|
||||
t.Fatalf("candidate owner = %#v, want current owner", candidates[0].Owner)
|
||||
}
|
||||
}
|
||||
|
||||
func pruneCandidatePaths(candidates []PruneCandidate) string {
|
||||
paths := make([]string, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
paths = append(paths, candidate.Path)
|
||||
}
|
||||
return strings.Join(paths, ",")
|
||||
}
|
||||
519
internal/state/shared_root.go
Normal file
519
internal/state/shared_root.go
Normal file
@@ -0,0 +1,519 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
type StateDocument struct {
|
||||
SingleOwner *DistributorState
|
||||
SharedRoot *SharedRootState
|
||||
}
|
||||
|
||||
type SharedRootState struct {
|
||||
SchemaVersion int
|
||||
DistributorVersion string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
State StatePolicy
|
||||
Owners []OwnerRecord
|
||||
Outputs []SharedRootOutputFile
|
||||
}
|
||||
|
||||
type OwnerScope struct {
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
}
|
||||
|
||||
type OwnerRecord struct {
|
||||
Scope OwnerScope
|
||||
Reconciliation ReconciliationPolicy
|
||||
Source SourceState
|
||||
Links *LinkState
|
||||
}
|
||||
|
||||
type SharedRootOutputFile struct {
|
||||
Path string
|
||||
Kind string
|
||||
SourcePath string
|
||||
Transform string
|
||||
URL string
|
||||
SHA256 string
|
||||
Size int64
|
||||
Owner OwnerScope
|
||||
SourceID string
|
||||
SourceDigest string
|
||||
SourceCreated time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type PathOwnershipConflict struct {
|
||||
Path string
|
||||
Owner OwnerScope
|
||||
}
|
||||
|
||||
type rawSharedRootState struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
State *rawStatePolicy `json:"state"`
|
||||
Owners []rawOwnerRecord `json:"owners"`
|
||||
Outputs []rawSharedRootOutput `json:"outputs"`
|
||||
}
|
||||
|
||||
type rawOwnerRecord struct {
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
Reconciliation *rawReconciliationPolicy `json:"reconciliation"`
|
||||
Source *rawSourceState `json:"source"`
|
||||
Links *rawLinkState `json:"links"`
|
||||
}
|
||||
|
||||
type rawSharedRootOutput struct {
|
||||
Path *string `json:"path"`
|
||||
Kind *string `json:"kind"`
|
||||
SourcePath *string `json:"source_path"`
|
||||
Transform string `json:"transform"`
|
||||
URL string `json:"url"`
|
||||
SHA256 *string `json:"sha256"`
|
||||
Size *int64 `json:"size"`
|
||||
PipelineID *string `json:"pipeline_id"`
|
||||
DestinationID *string `json:"destination_id"`
|
||||
SourceID *string `json:"source_id"`
|
||||
SourceDigest *string `json:"source_digest"`
|
||||
SourceCreated *string `json:"source_created"`
|
||||
CreatedAt *string `json:"created_at"`
|
||||
UpdatedAt *string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func ParseDocument(data []byte) (StateDocument, error) {
|
||||
schemaVersion, err := parseSchemaVersion(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
if schemaVersion == SharedRootSchemaVersion {
|
||||
sharedRoot, err := ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{SharedRoot: &sharedRoot}, nil
|
||||
}
|
||||
singleOwner, err := Parse(data)
|
||||
if err != nil {
|
||||
return StateDocument{}, err
|
||||
}
|
||||
return StateDocument{SingleOwner: &singleOwner}, nil
|
||||
}
|
||||
|
||||
func parseSchemaVersion(data []byte) (int, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw struct {
|
||||
SchemaVersion *int `json:"schema_version"`
|
||||
}
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return 0, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
if raw.SchemaVersion == nil {
|
||||
return 0, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
return *raw.SchemaVersion, nil
|
||||
}
|
||||
|
||||
func ParseSharedRoot(data []byte) (SharedRootState, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
var raw rawSharedRootState
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return SharedRootState{}, fmt.Errorf("parse distributor state: %w", err)
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
return SharedRootState{}, fmt.Errorf("parse distributor state: trailing data")
|
||||
}
|
||||
state, err := parseSharedRootRaw(raw)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
if err := ValidateSharedRoot(state); err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseSharedRootRaw(raw rawSharedRootState) (SharedRootState, error) {
|
||||
if raw.SchemaVersion == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state schema_version is required")
|
||||
}
|
||||
state := SharedRootState{SchemaVersion: *raw.SchemaVersion}
|
||||
if state.SchemaVersion != SharedRootSchemaVersion {
|
||||
return SharedRootState{}, fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
||||
}
|
||||
state.DistributorVersion = raw.DistributorVersion
|
||||
createdAt, err := parseRequiredTime("state created_at", raw.CreatedAt)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime("state updated_at", raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.CreatedAt = createdAt
|
||||
state.UpdatedAt = updatedAt
|
||||
if raw.State == nil || raw.State.Mode == "" {
|
||||
return SharedRootState{}, fmt.Errorf("state state.mode is required")
|
||||
}
|
||||
state.State.Mode = raw.State.Mode
|
||||
if raw.Owners == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state owners is required")
|
||||
}
|
||||
owners, err := parseOwnerRecords(raw.Owners)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.Owners = owners
|
||||
if raw.Outputs == nil {
|
||||
return SharedRootState{}, fmt.Errorf("state outputs is required")
|
||||
}
|
||||
outputs, err := parseSharedRootOutputs(raw.Outputs)
|
||||
if err != nil {
|
||||
return SharedRootState{}, err
|
||||
}
|
||||
state.Outputs = outputs
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func parseOwnerRecords(rawOwners []rawOwnerRecord) ([]OwnerRecord, error) {
|
||||
owners := make([]OwnerRecord, 0, len(rawOwners))
|
||||
for index, raw := range rawOwners {
|
||||
owner, err := parseOwnerRecord(index, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
owners = append(owners, owner)
|
||||
}
|
||||
return owners, nil
|
||||
}
|
||||
|
||||
func parseOwnerRecord(index int, raw rawOwnerRecord) (OwnerRecord, error) {
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
||||
}
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].destination_id is required", index)
|
||||
}
|
||||
if raw.Reconciliation == nil || raw.Reconciliation.Mode == "" {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].reconciliation.mode is required", index)
|
||||
}
|
||||
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest is required", index)
|
||||
}
|
||||
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
||||
if err != nil {
|
||||
return OwnerRecord{}, fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
||||
}
|
||||
owner := OwnerRecord{
|
||||
Scope: OwnerScope{
|
||||
PipelineID: *raw.PipelineID,
|
||||
DestinationID: *raw.DestinationID,
|
||||
},
|
||||
Reconciliation: ReconciliationPolicy{Mode: raw.Reconciliation.Mode},
|
||||
Source: SourceState{Manifest: manifest},
|
||||
}
|
||||
if raw.Links != nil {
|
||||
owner.Links = &LinkState{PrimaryURL: raw.Links.PrimaryURL}
|
||||
}
|
||||
return owner, nil
|
||||
}
|
||||
|
||||
func parseSharedRootOutputs(rawOutputs []rawSharedRootOutput) ([]SharedRootOutputFile, error) {
|
||||
outputs := make([]SharedRootOutputFile, 0, len(rawOutputs))
|
||||
for index, raw := range rawOutputs {
|
||||
output, err := parseSharedRootOutput(index, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs, nil
|
||||
}
|
||||
|
||||
func parseSharedRootOutput(index int, raw rawSharedRootOutput) (SharedRootOutputFile, error) {
|
||||
if raw.Path == nil || *raw.Path == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||
}
|
||||
if raw.Kind == nil || *raw.Kind == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
||||
}
|
||||
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
||||
}
|
||||
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
||||
}
|
||||
if raw.Size == nil {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||
}
|
||||
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
if raw.SourceID == nil || *raw.SourceID == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_id is required", index)
|
||||
}
|
||||
if raw.SourceDigest == nil || *raw.SourceDigest == "" {
|
||||
return SharedRootOutputFile{}, fmt.Errorf("state outputs[%d].source_digest is required", index)
|
||||
}
|
||||
sourceCreated, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].source_created", index), raw.SourceCreated)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
createdAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].created_at", index), raw.CreatedAt)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
updatedAt, err := parseRequiredTime(fmt.Sprintf("state outputs[%d].updated_at", index), raw.UpdatedAt)
|
||||
if err != nil {
|
||||
return SharedRootOutputFile{}, err
|
||||
}
|
||||
return SharedRootOutputFile{
|
||||
Path: *raw.Path,
|
||||
Kind: *raw.Kind,
|
||||
SourcePath: *raw.SourcePath,
|
||||
Transform: raw.Transform,
|
||||
URL: raw.URL,
|
||||
SHA256: *raw.SHA256,
|
||||
Size: *raw.Size,
|
||||
Owner: OwnerScope{
|
||||
PipelineID: *raw.PipelineID,
|
||||
DestinationID: *raw.DestinationID,
|
||||
},
|
||||
SourceID: *raw.SourceID,
|
||||
SourceDigest: *raw.SourceDigest,
|
||||
SourceCreated: sourceCreated,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s SharedRootState) CreatedAtString() string {
|
||||
return s.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s SharedRootState) UpdatedAtString() string {
|
||||
return s.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) SourceCreatedString() string {
|
||||
return o.SourceCreated.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) CreatedAtString() string {
|
||||
return o.CreatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) UpdatedAtString() string {
|
||||
return o.UpdatedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (s SharedRootState) MarshalJSON() ([]byte, error) {
|
||||
type stateJSON struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
DistributorVersion string `json:"distributor_version,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
State StatePolicy `json:"state"`
|
||||
Owners []OwnerRecord `json:"owners"`
|
||||
Outputs []SharedRootOutputFile `json:"outputs"`
|
||||
}
|
||||
return json.Marshal(stateJSON{
|
||||
SchemaVersion: s.SchemaVersion,
|
||||
DistributorVersion: s.DistributorVersion,
|
||||
CreatedAt: s.CreatedAtString(),
|
||||
UpdatedAt: s.UpdatedAtString(),
|
||||
State: s.State,
|
||||
Owners: s.Owners,
|
||||
Outputs: s.Outputs,
|
||||
})
|
||||
}
|
||||
|
||||
func (o OwnerRecord) MarshalJSON() ([]byte, error) {
|
||||
type sourceJSON struct {
|
||||
Manifest bundle.Manifest `json:"manifest"`
|
||||
}
|
||||
type ownerJSON struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Reconciliation ReconciliationPolicy `json:"reconciliation"`
|
||||
Source sourceJSON `json:"source"`
|
||||
Links *LinkState `json:"links,omitempty"`
|
||||
}
|
||||
return json.Marshal(ownerJSON{
|
||||
PipelineID: o.Scope.PipelineID,
|
||||
DestinationID: o.Scope.DestinationID,
|
||||
Reconciliation: o.Reconciliation,
|
||||
Source: sourceJSON{Manifest: o.Source.Manifest},
|
||||
Links: o.Links,
|
||||
})
|
||||
}
|
||||
|
||||
func (o SharedRootOutputFile) MarshalJSON() ([]byte, error) {
|
||||
type outputJSON struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
SourcePath string `json:"source_path"`
|
||||
Transform string `json:"transform,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
SourceDigest string `json:"source_digest"`
|
||||
SourceCreated string `json:"source_created"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
return json.Marshal(outputJSON{
|
||||
Path: o.Path,
|
||||
Kind: o.Kind,
|
||||
SourcePath: o.SourcePath,
|
||||
Transform: o.Transform,
|
||||
URL: o.URL,
|
||||
SHA256: o.SHA256,
|
||||
Size: o.Size,
|
||||
PipelineID: o.Owner.PipelineID,
|
||||
DestinationID: o.Owner.DestinationID,
|
||||
SourceID: o.SourceID,
|
||||
SourceDigest: o.SourceDigest,
|
||||
SourceCreated: o.SourceCreatedString(),
|
||||
CreatedAt: o.CreatedAtString(),
|
||||
UpdatedAt: o.UpdatedAtString(),
|
||||
})
|
||||
}
|
||||
|
||||
func ValidateSharedRoot(s SharedRootState) error {
|
||||
if s.SchemaVersion != SharedRootSchemaVersion {
|
||||
return fmt.Errorf("state schema_version must be %d", SharedRootSchemaVersion)
|
||||
}
|
||||
if s.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state created_at is required")
|
||||
}
|
||||
if s.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state updated_at is required")
|
||||
}
|
||||
if s.State.Mode != StateModeSharedRoot {
|
||||
return fmt.Errorf("state state.mode must be %s", StateModeSharedRoot)
|
||||
}
|
||||
if s.Owners == nil {
|
||||
return fmt.Errorf("state owners is required")
|
||||
}
|
||||
owners := make(map[OwnerScope]OwnerRecord, len(s.Owners))
|
||||
for index, owner := range s.Owners {
|
||||
if err := validateOwnerRecord(index, owner); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := owners[owner.Scope]; exists {
|
||||
return fmt.Errorf("state owners[%d] duplicates owner %s/%s", index, owner.Scope.PipelineID, owner.Scope.DestinationID)
|
||||
}
|
||||
owners[owner.Scope] = owner
|
||||
}
|
||||
if s.Outputs == nil {
|
||||
return fmt.Errorf("state outputs is required")
|
||||
}
|
||||
seenPaths := make(map[string]struct{}, len(s.Outputs))
|
||||
for index, output := range s.Outputs {
|
||||
if err := validateSharedRootOutput(index, output, owners); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := seenPaths[output.Path]; exists {
|
||||
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||
}
|
||||
seenPaths[output.Path] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateOwnerRecord(index int, owner OwnerRecord) error {
|
||||
if owner.Scope.PipelineID == "" {
|
||||
return fmt.Errorf("state owners[%d].pipeline_id is required", index)
|
||||
}
|
||||
if owner.Scope.DestinationID == "" {
|
||||
return fmt.Errorf("state owners[%d].destination_id is required", index)
|
||||
}
|
||||
if owner.Reconciliation.Mode != config.ReconciliationModeReplace && owner.Reconciliation.Mode != config.ReconciliationModeMerge {
|
||||
return fmt.Errorf("state owners[%d].reconciliation.mode must be %s or %s", index, config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
if err := validateEmbeddedManifest(owner.Source.Manifest); err != nil {
|
||||
return fmt.Errorf("state owners[%d].source.manifest: %w", index, err)
|
||||
}
|
||||
if owner.Links != nil && owner.Links.PrimaryURL != "" {
|
||||
if err := link.ValidateHTTPURL(owner.Links.PrimaryURL); err != nil {
|
||||
return fmt.Errorf("state owners[%d].links.primary_url: %w", index, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSharedRootOutput(index int, output SharedRootOutputFile, owners map[OwnerScope]OwnerRecord) error {
|
||||
if err := storage.ValidatePath(output.Path); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
||||
}
|
||||
switch output.Kind {
|
||||
case OutputKindSource, OutputKindGenerated:
|
||||
default:
|
||||
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
||||
}
|
||||
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
||||
}
|
||||
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
||||
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||
}
|
||||
if output.URL != "" {
|
||||
if err := link.ValidateHTTPURL(output.URL); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].url: %w", index, err)
|
||||
}
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||
}
|
||||
if output.Size < 0 {
|
||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||
}
|
||||
if output.Owner.PipelineID == "" {
|
||||
return fmt.Errorf("state outputs[%d].pipeline_id is required", index)
|
||||
}
|
||||
if output.Owner.DestinationID == "" {
|
||||
return fmt.Errorf("state outputs[%d].destination_id is required", index)
|
||||
}
|
||||
if _, exists := owners[output.Owner]; !exists {
|
||||
return fmt.Errorf("state outputs[%d] references unknown owner %s/%s", index, output.Owner.PipelineID, output.Owner.DestinationID)
|
||||
}
|
||||
if output.SourceID == "" {
|
||||
return fmt.Errorf("state outputs[%d].source_id is required", index)
|
||||
}
|
||||
if err := bundle.ValidateDigest(output.SourceDigest); err != nil {
|
||||
return fmt.Errorf("state outputs[%d].source_digest: %w", index, err)
|
||||
}
|
||||
if output.SourceCreated.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].source_created is required", index)
|
||||
}
|
||||
if output.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
||||
}
|
||||
if output.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
320
internal/state/shared_root_test.go
Normal file
320
internal/state/shared_root_test.go
Normal file
@@ -0,0 +1,320 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
)
|
||||
|
||||
func TestParseDocumentHandlesSingleOwnerAndSharedRoot(t *testing.T) {
|
||||
singleOwner, err := ParseDocument([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(single owner) error = %v", err)
|
||||
}
|
||||
if singleOwner.SingleOwner == nil || singleOwner.SharedRoot != nil {
|
||||
t.Fatalf("single owner document = %#v", singleOwner)
|
||||
}
|
||||
|
||||
sharedRoot, err := ParseDocument([]byte(validSharedRootStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseDocument(shared root) error = %v", err)
|
||||
}
|
||||
if sharedRoot.SharedRoot == nil || sharedRoot.SingleOwner != nil {
|
||||
t.Fatalf("shared root document = %#v", sharedRoot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSharedRootState(t *testing.T) {
|
||||
state, err := ParseSharedRoot([]byte(validSharedRootStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseSharedRoot() error = %v", err)
|
||||
}
|
||||
if got, want := state.SchemaVersion, SharedRootSchemaVersion; got != want {
|
||||
t.Fatalf("schema version = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := state.CreatedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||
t.Fatalf("created_at = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := state.State.Mode, StateModeSharedRoot; got != want {
|
||||
t.Fatalf("state mode = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(state.Owners), 2; got != want {
|
||||
t.Fatalf("owner count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(state.Outputs), 2; got != want {
|
||||
t.Fatalf("output count = %d, want %d", got, want)
|
||||
}
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
manifest, ok := state.SourceManifest(scope)
|
||||
if !ok {
|
||||
t.Fatal("SourceManifest() ok = false, want true")
|
||||
}
|
||||
if got, want := manifest.ID, validManifest(t).ID; got != want {
|
||||
t.Fatalf("source manifest id = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSharedRootRejectsInvalidMetadata(t *testing.T) {
|
||||
tests := map[string]func(string) string{
|
||||
"schema": func(body string) string {
|
||||
return strings.Replace(body, `"schema_version": 3`, `"schema_version": 2`, 1)
|
||||
},
|
||||
"state mode": func(body string) string {
|
||||
return strings.Replace(body, `"mode": "shared_root"`, `"mode": "single_owner"`, 1)
|
||||
},
|
||||
"duplicate owner": func(body string) string {
|
||||
return strings.Replace(body, `"destination_id": "html"`, `"destination_id": "archive"`, 1)
|
||||
},
|
||||
"unknown output owner": func(body string) string {
|
||||
return strings.Replace(body, `"destination_id": "html",`, `"destination_id": "missing",`, 1)
|
||||
},
|
||||
"duplicate output": func(body string) string {
|
||||
return strings.Replace(body, `"path": "report.html"`, `"path": "report.md"`, 1)
|
||||
},
|
||||
"invalid source digest": func(body string) string {
|
||||
return strings.Replace(body, `"source_digest": "sha256:`, `"source_digest": "SHA256:`, 1)
|
||||
},
|
||||
"invalid owner link": func(body string) string {
|
||||
return strings.Replace(body, `"primary_url": "https://reports.example.com/archive/report.md"`, `"primary_url": "file:///tmp/report.md"`, 1)
|
||||
},
|
||||
}
|
||||
for name, mutate := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := ParseSharedRoot([]byte(mutate(validSharedRootStateJSON(t))))
|
||||
if err == nil {
|
||||
t.Fatal("ParseSharedRoot() error = nil, want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootMarshalNormalizesTimestamps(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
state.CreatedAt = time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60))
|
||||
state.UpdatedAt = state.CreatedAt
|
||||
state.Outputs[0].SourceCreated = state.CreatedAt
|
||||
state.Outputs[0].CreatedAt = state.CreatedAt
|
||||
state.Outputs[0].UpdatedAt = state.CreatedAt
|
||||
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`"created_at":"2026-05-30T11:12:00Z"`,
|
||||
`"updated_at":"2026-05-30T11:12:00Z"`,
|
||||
`"source_created":"2026-05-30T11:12:00Z"`,
|
||||
} {
|
||||
if !strings.Contains(string(data), want) {
|
||||
t.Fatalf("json = %s, want %s", data, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootOutputHelpers(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
html := CurrentOwnerScope("reports", "html")
|
||||
|
||||
if got, want := state.ManagedOutputPaths(archive), []string{"report.md"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("archive paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := state.AllManagedOutputPaths(), []string{"report.md", "report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("all paths = %#v, want %#v", got, want)
|
||||
}
|
||||
conflict, ok := state.PathOwnershipConflict(archive, []string{"report.html"})
|
||||
if !ok || conflict.Owner != html {
|
||||
t.Fatalf("conflict = %#v ok=%t, want html owner conflict", conflict, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingSharedRootOwnerOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
next, changed := RemoveMissingSharedRootOwnerOutputs(state, archive, []string{"report.md", "report.html"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingSharedRootOwnerOutputs() changed = false, want true")
|
||||
}
|
||||
if got, want := next.AllManagedOutputPaths(), []string{"report.html"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if _, ok := next.OutputOwner("report.html"); !ok {
|
||||
t.Fatal("report.html owner missing, want unrelated owner preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveMissingSharedRootOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
next, changed := RemoveMissingSharedRootOutputs(state, []string{"report.md", "report.html"})
|
||||
if !changed {
|
||||
t.Fatal("RemoveMissingSharedRootOutputs() changed = false, want true")
|
||||
}
|
||||
if got := next.AllManagedOutputPaths(); len(got) != 0 {
|
||||
t.Fatalf("paths = %#v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootProjectAndMergeOwnerOutputs(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
owner, ok := state.Owner(archive)
|
||||
if !ok {
|
||||
t.Fatal("Owner() ok = false, want true")
|
||||
}
|
||||
now := time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC)
|
||||
planned := ProjectSharedRootOutputs([]OutputProjection{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: validManifest(t).Files[0].SHA256,
|
||||
Size: validManifest(t).Files[0].Size,
|
||||
}, {
|
||||
Path: "summary.txt",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: validManifest(t).Files[1].SHA256,
|
||||
Size: validManifest(t).Files[1].Size,
|
||||
}}, state.Outputs, archive, validManifest(t), now)
|
||||
|
||||
merged, err := MergeOwnerOutputs(state, archive, owner, planned)
|
||||
if err != nil {
|
||||
t.Fatalf("MergeOwnerOutputs() error = %v", err)
|
||||
}
|
||||
if got, want := merged.AllManagedOutputPaths(), []string{"report.md", "report.html", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("merged paths = %#v, want %#v", got, want)
|
||||
}
|
||||
if merged.Outputs[0].CreatedAt.Equal(now) {
|
||||
t.Fatalf("merged updated output created_at = %s, want preserved timestamp", merged.Outputs[0].CreatedAt)
|
||||
}
|
||||
if !merged.Outputs[0].UpdatedAt.Equal(now) {
|
||||
t.Fatalf("merged updated output updated_at = %s, want %s", merged.Outputs[0].UpdatedAt, now)
|
||||
}
|
||||
|
||||
replaced, err := ReplaceOwnerOutputs(state, archive, owner, planned)
|
||||
if err != nil {
|
||||
t.Fatalf("ReplaceOwnerOutputs() error = %v", err)
|
||||
}
|
||||
if got, want := replaced.AllManagedOutputPaths(), []string{"report.html", "report.md", "summary.txt"}; strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("replaced paths = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedRootOwnerOutputHelpersRejectConflicts(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
archive := CurrentOwnerScope("reports", "archive")
|
||||
owner, ok := state.Owner(archive)
|
||||
if !ok {
|
||||
t.Fatal("Owner() ok = false, want true")
|
||||
}
|
||||
planned := []SharedRootOutputFile{{
|
||||
Path: "report.html",
|
||||
Kind: OutputKindSource,
|
||||
Owner: archive,
|
||||
CreatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
||||
UpdatedAt: time.Date(2026, 5, 30, 12, 30, 0, 0, time.UTC),
|
||||
}}
|
||||
|
||||
if _, err := MergeOwnerOutputs(state, archive, owner, planned); err == nil {
|
||||
t.Fatal("MergeOwnerOutputs() error = nil, want owner conflict")
|
||||
}
|
||||
if _, err := ReplaceOwnerOutputs(state, archive, owner, planned); err == nil {
|
||||
t.Fatal("ReplaceOwnerOutputs() error = nil, want owner conflict")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareSharedRootOwnerScopesCurrentOwner(t *testing.T) {
|
||||
state := validSharedRootState(t)
|
||||
source := validManifest(t)
|
||||
source.Created = source.Created.Add(time.Hour)
|
||||
scope := CurrentOwnerScope("reports", "archive")
|
||||
|
||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{SharedRoot: &state, HasContents: true})
|
||||
if comparison.Outcome != OutcomeDestinationOlder {
|
||||
t.Fatalf("comparison = %#v, want destination older for current owner", comparison)
|
||||
}
|
||||
|
||||
missing := CompareSharedRootOwner(source, CurrentOwnerScope("missing", "archive"), DestinationStatus{SharedRoot: &state, HasContents: true})
|
||||
if missing.Outcome != OutcomeDestinationAbsent {
|
||||
t.Fatalf("missing owner comparison = %#v, want destination absent", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareSharedRootOwnerAcceptsMatchingSingleOwnerState(t *testing.T) {
|
||||
singleOwner, err := Parse([]byte(validStateJSON(t)))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
}
|
||||
source := singleOwner.Source.Manifest
|
||||
source.Created = source.Created.Add(time.Hour)
|
||||
scope := CurrentOwnerScope(singleOwner.PipelineID, singleOwner.DestinationID)
|
||||
|
||||
comparison := CompareSharedRootOwner(source, scope, DestinationStatus{State: &singleOwner, HasContents: true})
|
||||
if comparison.Outcome != OutcomeDestinationOlder {
|
||||
t.Fatalf("comparison = %#v, want destination older", comparison)
|
||||
}
|
||||
}
|
||||
|
||||
func validSharedRootStateJSON(t *testing.T) string {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(validSharedRootState(t), "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared root state: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func validSharedRootState(t *testing.T) SharedRootState {
|
||||
t.Helper()
|
||||
source := validManifest(t)
|
||||
htmlSource := source
|
||||
htmlSource.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
return SharedRootState{
|
||||
SchemaVersion: SharedRootSchemaVersion,
|
||||
DistributorVersion: "dev",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: StatePolicy{Mode: StateModeSharedRoot},
|
||||
Owners: []OwnerRecord{{
|
||||
Scope: CurrentOwnerScope("reports", "archive"),
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: SourceState{Manifest: source},
|
||||
Links: &LinkState{PrimaryURL: "https://reports.example.com/archive/report.md"},
|
||||
}, {
|
||||
Scope: CurrentOwnerScope("reports", "html"),
|
||||
Reconciliation: ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
||||
Source: SourceState{Manifest: htmlSource},
|
||||
}},
|
||||
Outputs: []SharedRootOutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
Owner: CurrentOwnerScope("reports", "archive"),
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
Kind: OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
URL: "https://reports.example.com/html/report.html",
|
||||
SHA256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Size: 128,
|
||||
Owner: CurrentOwnerScope("reports", "html"),
|
||||
SourceID: htmlSource.ID,
|
||||
SourceDigest: htmlSource.Digest,
|
||||
SourceCreated: htmlSource.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/link"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
@@ -26,6 +27,18 @@ func Validate(s DistributorState) error {
|
||||
if s.PublishedAt.IsZero() {
|
||||
return fmt.Errorf("state published_at is required")
|
||||
}
|
||||
if s.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state created_at is required")
|
||||
}
|
||||
if s.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state updated_at is required")
|
||||
}
|
||||
if s.State.Mode != StateModeSingleOwner {
|
||||
return fmt.Errorf("state state.mode must be %s", StateModeSingleOwner)
|
||||
}
|
||||
if s.Reconciliation.Mode != config.ReconciliationModeReplace && s.Reconciliation.Mode != config.ReconciliationModeMerge {
|
||||
return fmt.Errorf("state reconciliation.mode must be %s or %s", config.ReconciliationModeReplace, config.ReconciliationModeMerge)
|
||||
}
|
||||
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
|
||||
return fmt.Errorf("state source.manifest: %w", err)
|
||||
}
|
||||
@@ -80,5 +93,11 @@ func validateOutput(index int, output OutputFile) error {
|
||||
if output.Size < 0 {
|
||||
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||
}
|
||||
if output.CreatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].created_at is required", index)
|
||||
}
|
||||
if output.UpdatedAt.IsZero() {
|
||||
return fmt.Errorf("state outputs[%d].updated_at is required", index)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ type Backend interface {
|
||||
Stat(ctx context.Context, path string) (Entry, error)
|
||||
Walk(ctx context.Context, prefix string, opts WalkOptions, fn WalkFunc) error
|
||||
HasAny(ctx context.Context, prefix string) (bool, error)
|
||||
DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts DeleteOptions) error
|
||||
DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts DeleteOptions) error
|
||||
DeletePrefix(ctx context.Context, prefix string, opts DeleteOptions) error
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ const (
|
||||
OpStat = "stat"
|
||||
OpWalk = "walk"
|
||||
OpHasAny = "has any"
|
||||
OpDeleteManagedOutputs = "delete managed outputs"
|
||||
OpDeleteManagedBundle = "delete managed bundle"
|
||||
OpDeletePrefix = "delete prefix"
|
||||
OpRegisterBackend = "register backend"
|
||||
|
||||
@@ -159,23 +159,35 @@ func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
return b.deleteManagedTargets(ctx, storage.OpDeleteManagedBundle, func() ([]string, error) {
|
||||
return storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
}, opts)
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||
return b.deleteManagedTargets(ctx, storage.OpDeleteManagedOutputs, func() ([]string, error) {
|
||||
return storage.ManagedOutputTargets(bundlePath, managedOutputPaths)
|
||||
}, opts)
|
||||
}
|
||||
|
||||
func (b *Backend) deleteManagedTargets(ctx context.Context, op string, targetsFunc func() ([]string, error), opts storage.DeleteOptions) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
targets, err := storage.ManagedBundleTargets(bundlePath, managedOutputPaths)
|
||||
targets, err := targetsFunc()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, target := range targets {
|
||||
if _, ok := b.dirs[target]; ok {
|
||||
return storage.NewError(storage.OpDeleteManagedBundle, backendName, target, storage.ErrUnsupported, nil)
|
||||
return storage.NewError(op, backendName, target, storage.ErrUnsupported, nil)
|
||||
}
|
||||
if !b.exists(target) {
|
||||
if opts.IgnoreMissing {
|
||||
continue
|
||||
}
|
||||
return storage.NewError(storage.OpDeleteManagedBundle, backendName, target, storage.ErrNotFound, nil)
|
||||
return storage.NewError(op, backendName, target, storage.ErrNotFound, nil)
|
||||
}
|
||||
delete(b.files, target)
|
||||
delete(b.symlinks, target)
|
||||
|
||||
@@ -50,10 +50,23 @@ func DisplayPath(path string) string {
|
||||
}
|
||||
|
||||
func ManagedBundleTargets(bundlePath string, managedOutputPaths []string) ([]string, error) {
|
||||
targets, err := ManagedOutputTargets(bundlePath, managedOutputPaths)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
statePath, err := StatePath(bundlePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets = append(targets, statePath)
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
func ManagedOutputTargets(bundlePath string, managedOutputPaths []string) ([]string, error) {
|
||||
if err := ValidatePrefix(bundlePath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets := make([]string, 0, len(managedOutputPaths)+1)
|
||||
targets := make([]string, 0, len(managedOutputPaths))
|
||||
for _, outputPath := range managedOutputPaths {
|
||||
target, err := Join(bundlePath, outputPath)
|
||||
if err != nil {
|
||||
@@ -61,11 +74,6 @@ func ManagedBundleTargets(bundlePath string, managedOutputPaths []string) ([]str
|
||||
}
|
||||
targets = append(targets, target)
|
||||
}
|
||||
statePath, err := StatePath(bundlePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets = append(targets, statePath)
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -132,6 +132,22 @@ func TestManagedBundleTargetsRejectsInvalidOutputPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedOutputTargetsOmitsStateFile(t *testing.T) {
|
||||
targets, err := ManagedOutputTargets("bundle", []string{"report.md", "nested/report.html"})
|
||||
if err != nil {
|
||||
t.Fatalf("ManagedOutputTargets() error = %v", err)
|
||||
}
|
||||
want := []string{"bundle/report.md", "bundle/nested/report.html"}
|
||||
if len(targets) != len(want) {
|
||||
t.Fatalf("targets = %v, want %v", targets, want)
|
||||
}
|
||||
for index := range want {
|
||||
if targets[index] != want[index] {
|
||||
t.Fatalf("targets = %v, want %v", targets, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSortsEntries(t *testing.T) {
|
||||
backend := walkBackend{
|
||||
entries: []Entry{
|
||||
@@ -215,6 +231,10 @@ func (b walkBackend) DeleteManagedBundle(context.Context, string, []string, Dele
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b walkBackend) DeleteManagedOutputs(context.Context, string, []string, DeleteOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b walkBackend) DeletePrefix(context.Context, string, DeleteOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
@@ -329,14 +330,19 @@ func WriteDestinationState(t testing.TB, root, relative string, manifest bundle.
|
||||
}
|
||||
|
||||
func DestinationState(manifest bundle.Manifest, opts DestinationStateOptions) state.DistributorState {
|
||||
publishedAt := defaultPublishedAt(opts.PublishedAt)
|
||||
return state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
DistributorVersion: opts.DistributorVersion,
|
||||
PipelineID: defaultString(opts.PipelineID, "reports"),
|
||||
DestinationID: defaultString(opts.DestinationID, "archive"),
|
||||
PublishedAt: defaultPublishedAt(opts.PublishedAt),
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
Outputs: sourceOutputs(manifest),
|
||||
Outputs: sourceOutputs(manifest, publishedAt),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -386,7 +392,7 @@ func sourceFiles(opts BundleOptions) []SourceFile {
|
||||
return files
|
||||
}
|
||||
|
||||
func sourceOutputs(manifest bundle.Manifest) []state.OutputFile {
|
||||
func sourceOutputs(manifest bundle.Manifest, publishedAt time.Time) []state.OutputFile {
|
||||
outputs := make([]state.OutputFile, 0, len(manifest.Files))
|
||||
for _, file := range manifest.Files {
|
||||
outputs = append(outputs, state.OutputFile{
|
||||
@@ -395,6 +401,8 @@ func sourceOutputs(manifest bundle.Manifest) []state.OutputFile {
|
||||
SourcePath: file.Path,
|
||||
SHA256: file.SHA256,
|
||||
Size: file.Size,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
})
|
||||
}
|
||||
return outputs
|
||||
|
||||
@@ -1,9 +1,104 @@
|
||||
// Package bundle provides producer-facing helpers for distributor source
|
||||
// bundle manifests.
|
||||
// bundles.
|
||||
//
|
||||
// A source bundle is a local directory containing a manifest.json file and the
|
||||
// files listed by that manifest. This package owns the public manifest model,
|
||||
// digest calculation, path validation, manifest parsing, manifest building,
|
||||
// local bundle writing, and local bundle validation used by Go producer
|
||||
// applications.
|
||||
// A source bundle is a local directory containing manifest.json and the files
|
||||
// listed by that manifest. Producer applications use this package when they
|
||||
// need to generate manifests, validate bundles locally, or write complete
|
||||
// bundle directories for distributor to discover, upload, or publish.
|
||||
//
|
||||
// # Bundle Contract
|
||||
//
|
||||
// The source manifest is the producer-to-distributor contract. It is named by
|
||||
// ManifestName, currently "manifest.json", and uses SchemaVersion, currently 1.
|
||||
// A Manifest contains:
|
||||
//
|
||||
// - SchemaVersion: the source manifest schema version.
|
||||
// - ID: the producer's stable bundle identifier.
|
||||
// - Digest: the canonical digest of the ordered file records.
|
||||
// - Created: an RFC3339 timestamp when marshaled to JSON.
|
||||
// - Files: an ordered list of ManifestFile records.
|
||||
//
|
||||
// Each ManifestFile records a slash-separated bundle-relative Path, a lowercase
|
||||
// sha256:<64 hex> SHA256 digest, and a byte Size. File order is significant for
|
||||
// the bundle digest and should be chosen deliberately by the producer. Explicit
|
||||
// file lists preserve caller order; scan mode sorts by slash-separated path.
|
||||
//
|
||||
// # Path Rules
|
||||
//
|
||||
// Public bundle paths are always slash-separated and relative to the bundle
|
||||
// root. ValidateSourcePath rejects empty paths, absolute paths, path traversal,
|
||||
// dot segments, backslashes, and reserved manifest/state paths. Source files
|
||||
// must be regular files; symlinks and other special files are rejected.
|
||||
//
|
||||
// BuildManifest with Scan true recursively scans Root, includes regular files
|
||||
// including dotfiles, excludes manifest.json and .distributor.json, rejects
|
||||
// symlinks, and sorts paths lexically. BuildManifest with Files uses exactly
|
||||
// the caller-provided paths and preserves their order. Exactly one selection
|
||||
// mode must be used.
|
||||
//
|
||||
// # Manifest Workflows
|
||||
//
|
||||
// BuildManifest reads existing files under a local root, calculates each
|
||||
// ManifestFile, defaults a zero Created value to the current UTC time, calculates
|
||||
// the bundle digest, and validates the result. WriteManifest writes
|
||||
// manifest.json and fails if it already exists unless WriteManifestOptions has
|
||||
// Overwrite set. LoadManifest reads and parses manifest.json. ParseManifest and
|
||||
// MarshalManifest are useful when an application stores or transmits manifest
|
||||
// bytes directly; MarshalManifest validates before writing deterministic,
|
||||
// indented JSON with a trailing newline.
|
||||
//
|
||||
// ValidateManifest checks manifest-only semantics, including schema version,
|
||||
// required fields, path safety, duplicate file paths, digest syntax, file sizes,
|
||||
// and bundle digest. ValidateBundle checks a supplied Manifest against local
|
||||
// files under a root, including existence, regular-file type, size, SHA-256
|
||||
// digest, path safety, and bundle digest.
|
||||
//
|
||||
// # Complete Bundle Writing
|
||||
//
|
||||
// WriteBundle is the most convenient producer workflow when source files live
|
||||
// outside the final bundle directory. It copies each BundleFile.SourcePath into
|
||||
// a staged bundle at BundleFile.Path, builds and writes a compliant manifest,
|
||||
// validates the staged bundle, and promotes it to WriteBundleOptions.Root.
|
||||
// Overwrite permits replacement of an existing bundle root using a best-effort
|
||||
// sibling temporary and backup strategy.
|
||||
//
|
||||
// # Digest Helpers
|
||||
//
|
||||
// FileDigest returns the sha256:<64 hex> digest for file bytes. BundleDigest
|
||||
// returns the canonical bundle digest for an ordered []ManifestFile.
|
||||
// CanonicalFilePayload returns the JSON payload used by BundleDigest, which is
|
||||
// mainly useful for tests and diagnostics. ValidateDigest checks digest syntax.
|
||||
//
|
||||
// Example: build and write a manifest for files already under a bundle root.
|
||||
//
|
||||
// root := "/var/lib/reports/daily-2026-06-06"
|
||||
// manifest, err := bundle.BuildManifest(bundle.BuildOptions{
|
||||
// Root: root,
|
||||
// ID: "reports.daily.2026-06-06",
|
||||
// Files: []string{"report.md", "summary.txt"},
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// if err := bundle.WriteManifest(root, manifest, bundle.WriteManifestOptions{}); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// if err := bundle.ValidateBundle(root, manifest); err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// Example: create a complete bundle from producer-generated files.
|
||||
//
|
||||
// manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{
|
||||
// Root: "/var/lib/distributor-source/daily-2026-06-06",
|
||||
// ID: "reports.daily.2026-06-06",
|
||||
// Files: []bundle.BundleFile{
|
||||
// {SourcePath: "/tmp/report.md", Path: "report.md"},
|
||||
// {SourcePath: "/tmp/summary.txt", Path: "summary.txt"},
|
||||
// },
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// _ = manifest
|
||||
package bundle
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -22,7 +23,6 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
uploadPath = "upload"
|
||||
runsPath = "runs"
|
||||
idempotencyKeyHeader = "Idempotency-Key"
|
||||
defaultHTTPTimeout = 30 * time.Second
|
||||
@@ -34,6 +34,8 @@ const (
|
||||
redactedSecret = "[redacted]"
|
||||
)
|
||||
|
||||
var pipelineIDPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`)
|
||||
|
||||
func NewClient(opts ClientOptions) (*Client, error) {
|
||||
endpoint, err := cleanEndpoint(opts.Endpoint)
|
||||
if err != nil {
|
||||
@@ -65,6 +67,9 @@ func (c *Client) UploadBundle(ctx context.Context, opts UploadBundleOptions) (Re
|
||||
if opts.Validate && opts.DisableValidation {
|
||||
return Result{}, fmt.Errorf("validate and disable validation cannot both be set")
|
||||
}
|
||||
if err := validatePipelineID(opts.PipelineID); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if opts.Root == "" {
|
||||
return Result{}, fmt.Errorf("root is required")
|
||||
}
|
||||
@@ -85,7 +90,7 @@ func (c *Client) UploadBundle(ctx context.Context, opts UploadBundleOptions) (Re
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return c.uploadArchive(ctx, archive, key)
|
||||
return c.uploadArchive(ctx, opts.PipelineID, archive, key)
|
||||
}
|
||||
|
||||
func (c *Client) UploadFiles(ctx context.Context, opts UploadFilesOptions) (Result, error) {
|
||||
@@ -95,6 +100,9 @@ func (c *Client) UploadFiles(ctx context.Context, opts UploadFilesOptions) (Resu
|
||||
if opts.Validate && opts.DisableValidation {
|
||||
return Result{}, fmt.Errorf("validate and disable validation cannot both be set")
|
||||
}
|
||||
if err := validatePipelineID(opts.PipelineID); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if opts.ID == "" {
|
||||
return Result{}, fmt.Errorf("id is required")
|
||||
}
|
||||
@@ -131,7 +139,7 @@ func (c *Client) UploadFiles(ctx context.Context, opts UploadFilesOptions) (Resu
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return c.uploadArchive(ctx, archive, key)
|
||||
return c.uploadArchive(ctx, opts.PipelineID, archive, key)
|
||||
}
|
||||
|
||||
func (c *Client) Status(ctx context.Context, runID string) (RunStatus, error) {
|
||||
@@ -167,7 +175,7 @@ func (c *Client) Status(ctx context.Context, runID string) (RunStatus, error) {
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func (c *Client) uploadArchive(ctx context.Context, archive []byte, idempotencyKey string) (Result, error) {
|
||||
func (c *Client) uploadArchive(ctx context.Context, pipelineID string, archive []byte, idempotencyKey string) (Result, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
@@ -176,7 +184,7 @@ func (c *Client) uploadArchive(ctx context.Context, archive []byte, idempotencyK
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
result, retry, err := c.uploadAttempt(ctx, archive, idempotencyKey)
|
||||
result, retry, err := c.uploadAttempt(ctx, pipelineID, archive, idempotencyKey)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
@@ -191,8 +199,8 @@ func (c *Client) uploadArchive(ctx context.Context, archive []byte, idempotencyK
|
||||
return Result{}, lastErr
|
||||
}
|
||||
|
||||
func (c *Client) uploadAttempt(ctx context.Context, archive []byte, idempotencyKey string) (Result, bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.uploadURL(), bytes.NewReader(archive))
|
||||
func (c *Client) uploadAttempt(ctx context.Context, pipelineID string, archive []byte, idempotencyKey string) (Result, bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.uploadURL(pipelineID), bytes.NewReader(archive))
|
||||
if err != nil {
|
||||
return Result{}, false, c.redactError(err)
|
||||
}
|
||||
@@ -227,8 +235,8 @@ func (c *Client) authorize(request *http.Request) {
|
||||
request.Header.Set("Authorization", authorizationPrefix+c.token)
|
||||
}
|
||||
|
||||
func (c *Client) uploadURL() string {
|
||||
return joinEndpointPath(c.endpoint, uploadPath)
|
||||
func (c *Client) uploadURL(pipelineID string) string {
|
||||
return joinEndpointPath(c.endpoint, "v1", "pipelines", pipelineID, "upload")
|
||||
}
|
||||
|
||||
func (c *Client) statusURL(runID string) string {
|
||||
@@ -343,6 +351,16 @@ func uploadIdempotencyKey(value string) (string, error) {
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func validatePipelineID(value string) error {
|
||||
if value == "" {
|
||||
return fmt.Errorf("pipeline id is required")
|
||||
}
|
||||
if !pipelineIDPattern.MatchString(value) {
|
||||
return fmt.Errorf("pipeline id must be a slug-like identifier")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateIdempotencyKey(value string) error {
|
||||
if value == "" {
|
||||
return fmt.Errorf("idempotency key is required")
|
||||
|
||||
@@ -47,7 +47,7 @@ func TestNewClientValidatesOptions(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
if got, want := client.uploadURL(), "http://127.0.0.1:8080/base/upload"; got != want {
|
||||
if got, want := client.uploadURL("reports.daily"), "http://127.0.0.1:8080/base/v1/pipelines/reports.daily/upload"; got != want {
|
||||
t.Fatalf("upload URL = %q, want %q", got, want)
|
||||
}
|
||||
if client.httpClient == nil || client.httpClient.Timeout == 0 {
|
||||
@@ -65,7 +65,7 @@ func TestUploadBundleSendsCallerKeyAndManifestArchive(t *testing.T) {
|
||||
}
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got, want := r.URL.Path, "/upload"; got != want {
|
||||
if got, want := r.URL.Path, "/v1/pipelines/reports.daily/upload"; got != want {
|
||||
t.Fatalf("path = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := r.Header.Get("Authorization"), "Bearer secret-token"; got != want {
|
||||
@@ -93,6 +93,7 @@ func TestUploadBundleSendsCallerKeyAndManifestArchive(t *testing.T) {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
result, err := client.UploadBundle(context.Background(), UploadBundleOptions{
|
||||
PipelineID: "reports.daily",
|
||||
Root: root,
|
||||
IdempotencyKey: "producer.retry:one",
|
||||
})
|
||||
@@ -112,6 +113,9 @@ func TestUploadFilesBuildsTemporaryBundleWithoutTouchingSources(t *testing.T) {
|
||||
}
|
||||
tempDir := t.TempDir()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got, want := r.URL.Path, "/v1/pipelines/reports.files/upload"; got != want {
|
||||
t.Fatalf("path = %q, want %q", got, want)
|
||||
}
|
||||
entries := readArchiveEntries(t, r.Body)
|
||||
if got := string(entries["manifest.json"]); !strings.Contains(got, `"id": "reports.from.files"`) {
|
||||
t.Fatalf("manifest = %s, want uploaded id", got)
|
||||
@@ -131,6 +135,7 @@ func TestUploadFilesBuildsTemporaryBundleWithoutTouchingSources(t *testing.T) {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
_, err = client.UploadFiles(context.Background(), UploadFilesOptions{
|
||||
PipelineID: "reports.files",
|
||||
ID: "reports.from.files",
|
||||
Files: []sourcebundle.BundleFile{{
|
||||
SourcePath: sourcePath,
|
||||
@@ -153,6 +158,77 @@ func TestUploadFilesBuildsTemporaryBundleWithoutTouchingSources(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMethodsRequirePipelineIDBeforeLocalWork(t *testing.T) {
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
t.Fatal("server should not receive request")
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewClient(ClientOptions{Endpoint: server.URL, Token: "secret", HTTPClient: server.Client()})
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
|
||||
missingRoot := filepath.Join(t.TempDir(), "missing")
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: missingRoot}); err == nil || !strings.Contains(err.Error(), "pipeline id is required") {
|
||||
t.Fatalf("UploadBundle() error = %v, want missing pipeline id", err)
|
||||
}
|
||||
|
||||
tempDir := t.TempDir()
|
||||
sourcePath := filepath.Join(t.TempDir(), "report.md")
|
||||
if err := os.WriteFile(sourcePath, []byte("data"), 0o600); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
if _, err := client.UploadFiles(context.Background(), UploadFilesOptions{
|
||||
ID: "reports.from.files",
|
||||
Files: []sourcebundle.BundleFile{{
|
||||
SourcePath: sourcePath,
|
||||
Path: "report.md",
|
||||
}},
|
||||
TempDir: tempDir,
|
||||
}); err == nil || !strings.Contains(err.Error(), "pipeline id is required") {
|
||||
t.Fatalf("UploadFiles() error = %v, want missing pipeline id", err)
|
||||
}
|
||||
entries, err := os.ReadDir(tempDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read temp dir: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("temp dir entries = %d, want no local bundle work", len(entries))
|
||||
}
|
||||
if got := requests.Load(); got != 0 {
|
||||
t.Fatalf("requests = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMethodsRejectInvalidPipelineIDBeforeHTTPRequest(t *testing.T) {
|
||||
root := writeTestBundle(t, "reports.daily", []testFile{{path: "report.md", data: "data"}})
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
t.Fatal("server should not receive request")
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewClient(ClientOptions{Endpoint: server.URL, Token: "secret", HTTPClient: server.Client()})
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
for _, pipelineID := range []string{".reports", "reports/daily", "reports daily"} {
|
||||
t.Run(pipelineID, func(t *testing.T) {
|
||||
_, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: pipelineID, Root: root})
|
||||
if err == nil || !strings.Contains(err.Error(), "pipeline id must be a slug-like identifier") {
|
||||
t.Fatalf("UploadBundle() error = %v, want invalid pipeline id", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if got := requests.Load(); got != 0 {
|
||||
t.Fatalf("requests = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadBundleValidationFailurePreventsHTTPRequest(t *testing.T) {
|
||||
root := writeTestBundle(t, "reports.daily", []testFile{{path: "report.md", data: "original"}})
|
||||
if err := os.WriteFile(filepath.Join(root, "report.md"), []byte("changed"), 0o600); err != nil {
|
||||
@@ -169,7 +245,7 @@ func TestUploadBundleValidationFailurePreventsHTTPRequest(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root}); err == nil {
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root}); err == nil {
|
||||
t.Fatal("UploadBundle() error = nil, want validation error")
|
||||
}
|
||||
if got := requests.Load(); got != 0 {
|
||||
@@ -193,7 +269,7 @@ func TestUploadBundleCanDisableLocalValidation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, DisableValidation: true}); err != nil {
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, DisableValidation: true}); err != nil {
|
||||
t.Fatalf("UploadBundle() error = %v", err)
|
||||
}
|
||||
if got := requests.Load(); got != 1 {
|
||||
@@ -206,6 +282,9 @@ func TestGeneratedIdempotencyKeyIsReusedAcrossRetry(t *testing.T) {
|
||||
var attempts atomic.Int64
|
||||
var keys []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got, want := r.URL.Path, "/v1/pipelines/reports/upload"; got != want {
|
||||
t.Fatalf("path = %q, want %q", got, want)
|
||||
}
|
||||
keys = append(keys, r.Header.Get(idempotencyKeyHeader))
|
||||
if attempts.Add(1) == 1 {
|
||||
writeJSONError(w, http.StatusServiceUnavailable, "busy", false)
|
||||
@@ -224,7 +303,7 @@ func TestGeneratedIdempotencyKeyIsReusedAcrossRetry(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root}); err != nil {
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root}); err != nil {
|
||||
t.Fatalf("UploadBundle() error = %v", err)
|
||||
}
|
||||
if got, want := attempts.Load(), int64(2); got != want {
|
||||
@@ -276,7 +355,7 @@ func TestUploadResponseParsingAndNoRetryStatuses(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
_, err = client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, IdempotencyKey: "key"})
|
||||
_, err = client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "key"})
|
||||
if err == nil {
|
||||
t.Fatal("UploadBundle() error = nil, want error")
|
||||
}
|
||||
@@ -309,7 +388,7 @@ func TestTokenRedactedFromHTTPError(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
_, err = client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, IdempotencyKey: "key"})
|
||||
_, err = client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "key"})
|
||||
if err == nil {
|
||||
t.Fatal("UploadBundle() error = nil, want error")
|
||||
}
|
||||
@@ -330,6 +409,9 @@ func TestNetworkRetryUsesSameIdempotencyKey(t *testing.T) {
|
||||
Token: "secret",
|
||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
keys = append(keys, request.Header.Get(idempotencyKeyHeader))
|
||||
if got, want := request.URL.Path, "/v1/pipelines/reports/upload"; got != want {
|
||||
t.Fatalf("path = %q, want %q", got, want)
|
||||
}
|
||||
if attempts.Add(1) == 1 {
|
||||
return nil, temporaryNetworkError{}
|
||||
}
|
||||
@@ -346,7 +428,7 @@ func TestNetworkRetryUsesSameIdempotencyKey(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
result, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, IdempotencyKey: "network-retry"})
|
||||
result, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "network-retry"})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadBundle() error = %v", err)
|
||||
}
|
||||
@@ -381,7 +463,7 @@ func TestContextCancellationDuringRetryBackoff(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
_, err = client.UploadBundle(ctx, UploadBundleOptions{Root: root, IdempotencyKey: "cancel"})
|
||||
_, err = client.UploadBundle(ctx, UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "cancel"})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("UploadBundle() error = %v, want context.Canceled", err)
|
||||
}
|
||||
@@ -446,7 +528,7 @@ func TestInvalidCallerIdempotencyKeyPreventsHTTPRequest(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewClient() error = %v", err)
|
||||
}
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{Root: root, IdempotencyKey: "bad key"}); err == nil {
|
||||
if _, err := client.UploadBundle(context.Background(), UploadBundleOptions{PipelineID: "reports", Root: root, IdempotencyKey: "bad key"}); err == nil {
|
||||
t.Fatal("UploadBundle() error = nil, want invalid key error")
|
||||
}
|
||||
if got := requests.Load(); got != 0 {
|
||||
|
||||
136
pkg/upload/doc.go
Normal file
136
pkg/upload/doc.go
Normal file
@@ -0,0 +1,136 @@
|
||||
// Package upload provides producer-facing helpers for submitting distributor
|
||||
// source bundles to the HTTP upload API.
|
||||
//
|
||||
// The package is intended for Go producer applications that already create
|
||||
// reports or other Markdown bundle contents and want to hand those bundles to a
|
||||
// running distributor server. It builds on pkg/bundle for manifest generation,
|
||||
// path validation, digest calculation, local bundle writing, and local bundle
|
||||
// validation. It does not expose distributor internals, server configuration,
|
||||
// storage backends, destination state, or publish behavior.
|
||||
//
|
||||
// # Client Construction
|
||||
//
|
||||
// NewClient creates a Client from ClientOptions. Endpoint is required and must
|
||||
// be an http or https distributor server base URL without userinfo, query, or
|
||||
// fragment. The client derives /v1/pipelines/<pipeline-id>/upload for
|
||||
// submissions and /runs/<run-id> for status checks. Token is required and is
|
||||
// sent as Authorization: Bearer <token>. Token values are redacted from errors
|
||||
// produced by the client.
|
||||
//
|
||||
// HTTPClient is optional. When omitted, the package uses a client with a
|
||||
// conservative timeout. Retry is optional; zero values select safe defaults.
|
||||
// RetryOptions.MaxAttempts, BaseDelay, and MaxDelay must not be negative, and
|
||||
// MaxDelay must be greater than or equal to BaseDelay.
|
||||
//
|
||||
// # Upload Workflows
|
||||
//
|
||||
// UploadBundle uploads an existing local source bundle root to the configured
|
||||
// PipelineID. PipelineID is required and must match the server's slug-like
|
||||
// pipeline id syntax. The root must contain manifest.json. By default,
|
||||
// UploadBundle loads the manifest and validates the complete local bundle with
|
||||
// pkg/bundle before making any HTTP request. The generated gzip-compressed tar
|
||||
// archive contains manifest.json and exactly the manifest-listed files;
|
||||
// unlisted files are not uploaded.
|
||||
//
|
||||
// UploadFiles is the convenience workflow for producer applications that have
|
||||
// generated files but have not yet assembled a bundle directory. PipelineID is
|
||||
// required and selects the configured distributor workflow. UploadFiles uses
|
||||
// pkg/bundle to create a temporary complete bundle from explicit
|
||||
// bundle.BundleFile values, validates it by default, archives it, uploads it,
|
||||
// and removes temporary files when the call returns. UploadFiles does not write
|
||||
// into producer source directories. A zero Created timestamp follows
|
||||
// pkg/bundle defaulting behavior.
|
||||
//
|
||||
// The producer contract has four separate identifiers: the bearer token
|
||||
// authenticates the client, PipelineID selects the distributor workflow, the
|
||||
// source manifest ID identifies the logical artifact within that workflow, and
|
||||
// IdempotencyKey identifies one producer run and retry group.
|
||||
//
|
||||
// Validation is enabled by default. Set DisableValidation when the application
|
||||
// has already performed equivalent local validation and wants to skip the
|
||||
// package's validation step. Validate and DisableValidation must not both be
|
||||
// true.
|
||||
//
|
||||
// # Idempotency And Retry
|
||||
//
|
||||
// Every upload request includes Idempotency-Key. If UploadBundleOptions or
|
||||
// UploadFilesOptions provides IdempotencyKey, the client validates and uses
|
||||
// that value. Otherwise, it generates a random 128-bit lowercase hexadecimal
|
||||
// key once for that upload operation and reuses it for all retries from that
|
||||
// call.
|
||||
//
|
||||
// Generated idempotency keys are useful for retrying transient failures within
|
||||
// a single process call. Producers that need cross-process retry safety should
|
||||
// provide their own stable key, such as a key derived from the producer job id
|
||||
// or report id. Valid keys are non-empty ASCII strings up to 128 bytes using
|
||||
// letters, digits, '.', '_', '-', and ':'.
|
||||
//
|
||||
// The client retries only safe cases: 503 Service Unavailable, temporary
|
||||
// network errors, and ambiguous mid-upload failures. Retries use the same
|
||||
// idempotency key and replayable gzip archive body. The client does not retry
|
||||
// 400, 401, 409, 413, 415, or any response after 202 Accepted. Context
|
||||
// cancellation is honored before each attempt and while waiting between
|
||||
// retries.
|
||||
//
|
||||
// # Results And Errors
|
||||
//
|
||||
// Result represents upload admission. A successful UploadBundle or UploadFiles
|
||||
// call means the server accepted the upload and returned a run id; it does not
|
||||
// mean the asynchronous distribution run has finished successfully.
|
||||
//
|
||||
// Status fetches the current server status for a run id and returns RunStatus.
|
||||
// This is a separate polling helper; upload calls do not wait for publication
|
||||
// completion.
|
||||
//
|
||||
// Non-2xx upload and status responses return *HTTPError when the server status
|
||||
// can be represented as an HTTP failure. HTTPError includes the numeric status
|
||||
// code, HTTP status string, response message, and server retryable flag when
|
||||
// present. A 409 Conflict response is returned as *IdempotencyConflictError,
|
||||
// which wraps HTTPError and can be detected with errors.As.
|
||||
//
|
||||
// Example: upload producer files with a stable idempotency key.
|
||||
//
|
||||
// ctx := context.Background()
|
||||
// client, err := upload.NewClient(upload.ClientOptions{
|
||||
// Endpoint: "https://distributor.example.com",
|
||||
// Token: os.Getenv("DISTRIBUTOR_UPLOAD_TOKEN"),
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
|
||||
// PipelineID: "reports.daily",
|
||||
// ID: "reports.daily.2026-06-06",
|
||||
// IdempotencyKey: "reports.daily.2026-06-06",
|
||||
// Files: []bundle.BundleFile{
|
||||
// {SourcePath: "/tmp/report.md", Path: "report.md"},
|
||||
// {SourcePath: "/tmp/summary.txt", Path: "summary.txt"},
|
||||
// },
|
||||
// })
|
||||
// if err != nil {
|
||||
// var conflict *upload.IdempotencyConflictError
|
||||
// if errors.As(err, &conflict) {
|
||||
// return fmt.Errorf("upload conflicts with an earlier different bundle: %w", err)
|
||||
// }
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// status, err := client.Status(ctx, result.RunID)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// _ = status
|
||||
//
|
||||
// Example: upload an existing bundle root.
|
||||
//
|
||||
// result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
|
||||
// PipelineID: "reports.daily",
|
||||
// Root: "/var/lib/reports/daily-2026-06-06",
|
||||
// IdempotencyKey: "reports.daily.2026-06-06",
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// _ = result
|
||||
package upload
|
||||
@@ -30,6 +30,7 @@ type RetryOptions struct {
|
||||
}
|
||||
|
||||
type UploadBundleOptions struct {
|
||||
PipelineID string
|
||||
Root string
|
||||
Validate bool
|
||||
DisableValidation bool
|
||||
@@ -37,6 +38,7 @@ type UploadBundleOptions struct {
|
||||
}
|
||||
|
||||
type UploadFilesOptions struct {
|
||||
PipelineID string
|
||||
ID string
|
||||
Created time.Time
|
||||
Files []bundle.BundleFile
|
||||
|
||||
Reference in New Issue
Block a user