Expose managed output pruning
This commit is contained in:
44
docs/cli.md
44
docs/cli.md
@@ -22,6 +22,7 @@ distributor help
|
||||
distributor version [--format text|json]
|
||||
distributor run [--config <path>] [--dry-run] [--force] [--format text|json]
|
||||
distributor reconcile-state --config <path> --pipeline <id> --destination <id> [--all-owners] [--dry-run] [--format text|json]
|
||||
distributor prune --config <path> --pipeline <id> --destination <id> (--dry-run|--apply) [--format text|json]
|
||||
distributor serve [--config <path>]
|
||||
distributor validate [--format text|json] <path>
|
||||
distributor validate --config <path> --pipeline <id> [--bundle <path>] [--format text|json]
|
||||
@@ -35,6 +36,7 @@ distributor manifest create --id <bundle-id> [options] <bundle-path>
|
||||
- `version` prints the application name and version.
|
||||
- `run` executes configured pipelines against their destinations.
|
||||
- `reconcile-state` repairs destination state records for missing managed outputs.
|
||||
- `prune` deletes managed outputs selected by the destination retention policy when `--apply` is supplied.
|
||||
- `serve` starts the authenticated HTTP upload API defined by the configuration file.
|
||||
- `validate` checks a local bundle path or a configured source bundle.
|
||||
- `inspect` reports manifest and file metadata for a local bundle path or a configured source bundle.
|
||||
@@ -48,7 +50,7 @@ distributor manifest create --id <bundle-id> [options] <bundle-path>
|
||||
|
||||
### Common Output Format
|
||||
|
||||
`--format text|json` is supported by `version`, `run`, `reconcile-state`, `validate`, `inspect`, and `manifest create`.
|
||||
`--format text|json` is supported by `version`, `run`, `reconcile-state`, `prune`, `validate`, `inspect`, and `manifest create`.
|
||||
|
||||
- `text` is the default human-readable output.
|
||||
- `json` emits one JSON document for successful command execution.
|
||||
@@ -90,6 +92,21 @@ distributor reconcile-state --config <path> --pipeline <id> --destination <id> [
|
||||
|
||||
Without `--dry-run`, `reconcile-state` applies state repair by removing records for managed outputs that no longer exist in storage. It reports unmanaged entries but does not delete destination files, adopt unmanaged files, or validate output digests. The command accepts no positional arguments.
|
||||
|
||||
### `prune`
|
||||
|
||||
```sh
|
||||
distributor prune --config <path> --pipeline <id> --destination <id> (--dry-run|--apply) [--format text|json]
|
||||
```
|
||||
|
||||
- `--config <path>` loads the pipeline configuration and is required.
|
||||
- `--pipeline <id>` selects the pipeline used to identify the destination root and owner scope.
|
||||
- `--destination <id>` selects the destination root and owner scope.
|
||||
- `--dry-run` reports planned managed-output deletes without deleting outputs or rewriting `.distributor.json`.
|
||||
- `--apply` deletes planned managed outputs and rewrites `.distributor.json` after confirmed deletes.
|
||||
- `--format text|json` selects human-readable or machine-readable output.
|
||||
|
||||
Exactly one of `--dry-run` or `--apply` is required. The command uses only the selected destination's configured `retention.prune` policy; it does not accept one-off retention overrides. Apply mode deletes only planned managed output paths, preserves unmanaged files, and preserves `.distributor.json` even when no managed outputs remain.
|
||||
|
||||
### `serve`
|
||||
|
||||
```sh
|
||||
@@ -214,6 +231,30 @@ go run ./cmd/distributor reconcile-state \
|
||||
|
||||
Use `--all-owners` only for shared-root destination state when all owners inside the selected root should be repaired.
|
||||
|
||||
### Prune Managed Outputs
|
||||
|
||||
Preview managed outputs selected by the configured retention policy:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune \
|
||||
--config examples/local-publish.yml \
|
||||
--pipeline example-source-bundle \
|
||||
--destination local-archive \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
Apply after reviewing the report:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune \
|
||||
--config examples/local-publish.yml \
|
||||
--pipeline example-source-bundle \
|
||||
--destination local-archive \
|
||||
--apply
|
||||
```
|
||||
|
||||
Use `--format json` when automation needs structured prune results.
|
||||
|
||||
### Run HTML And Fan-Out Examples
|
||||
|
||||
```sh
|
||||
@@ -258,6 +299,7 @@ Text output is optimized for direct operator use. JSON output is optimized for a
|
||||
- Use `inspect --format json` when automation needs manifest metadata, normalized file details, or checksum information.
|
||||
- Use `run --dry-run` before publishing to review destination actions.
|
||||
- Use `reconcile-state --dry-run` to inspect missing managed output records before repairing destination state.
|
||||
- Use `prune --dry-run` before `prune --apply` to review configured retention deletes.
|
||||
- Use [Configuration](config.md) for schema and default details.
|
||||
- Use [Troubleshooting](troubleshooting.md) for common errors and corrective action.
|
||||
- Use [Operations](operations.md) for HTTP upload operation, state files, and recovery workflows.
|
||||
|
||||
@@ -17,7 +17,7 @@ YAML decoding rejects unknown fields. Defaults are applied after decoding and be
|
||||
Runtime backend support is command-specific:
|
||||
|
||||
- `run`, `validate --config`, and `inspect --config` execute `local`, `ssh`, and `s3` sources.
|
||||
- `run` executes `local`, `ssh`, and `s3` destinations.
|
||||
- `run`, `reconcile-state`, and `prune` execute `local`, `ssh`, and `s3` destinations.
|
||||
- `serve` uses `http_upload` sources through the HTTP upload API and publishes to configured `local`, `ssh`, and `s3` destinations.
|
||||
- `http_upload` is valid only as a source backend.
|
||||
|
||||
@@ -460,12 +460,14 @@ retention:
|
||||
```
|
||||
|
||||
- `retention.prune.enabled`: optional boolean. Default is `false`.
|
||||
- `retention.prune.older_than`: optional duration. When pruning is enabled, outputs older than this duration are eligible in prune planning.
|
||||
- `retention.prune.older_than`: optional duration. When pruning is enabled, outputs older than this duration are eligible for pruning.
|
||||
- `retention.prune.keep_latest`: optional non-negative integer. When pruning is enabled, this many newest managed outputs are preserved before age-based pruning is considered.
|
||||
|
||||
When `retention.prune.enabled` is `true`, at least one of `older_than` or `keep_latest` is required. `older_than` must be greater than zero, and `keep_latest` must be zero or greater.
|
||||
|
||||
Prune planning uses managed output `updated_at` timestamps from destination state. It is owner-scoped for shared-root state. The current implementation parses, validates, and plans pruning from this policy; it does not delete destination files.
|
||||
Pruning uses managed output `updated_at` timestamps from destination state. If both `keep_latest` and `older_than` are set, the newest `keep_latest` outputs are preserved first, then age-based pruning is applied to the remaining managed outputs.
|
||||
|
||||
The `prune` command is owner-scoped for shared-root state. `prune --dry-run` reports selected managed outputs without writing. `prune --apply` deletes only selected managed output paths and rewrites destination state after confirmed deletes. It does not delete unmanaged files or `.distributor.json`, and it does not run automatically after `run`.
|
||||
|
||||
## Transfer Policy
|
||||
|
||||
|
||||
@@ -126,6 +126,14 @@ For shared-root state, repair is scoped to the selected owner by default. With `
|
||||
|
||||
State repair does not validate output digests, delete destination files, adopt unmanaged entries, or rewrite invalid or mismatched state.
|
||||
|
||||
## Prune Semantics
|
||||
|
||||
`distributor prune` can delete managed outputs selected by the configured destination retention policy. It uses the configured pipeline and destination selector to open one destination root and reads that root's `.distributor.json`.
|
||||
|
||||
For single-owner state, the state `pipeline_id` and `destination_id` must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
|
||||
|
||||
Prune planning uses output `updated_at` timestamps. `prune --dry-run` reports planned managed-output deletes without deleting files or rewriting state. `prune --apply` deletes only planned managed output paths, removes confirmed deleted records from valid state, and updates the state timestamp. It does not delete unmanaged files or `.distributor.json`.
|
||||
|
||||
## Compatibility
|
||||
|
||||
`distributor` can read schema version `1` destination state for compatibility. When v1 state is read, it is treated as single-owner state with `reconciliation.mode: replace`. Missing top-level `created_at` and `updated_at` are inferred from `published_at`, and missing per-output timestamps are also inferred from `published_at`.
|
||||
|
||||
@@ -16,7 +16,7 @@ Outputs include `RunReport`, `ReconcileStateReport`, `PrunePlanReport`, `PruneRe
|
||||
|
||||
`internal/app` wires packages together but does not own manifest validation rules, destination state comparison, storage path rules, publish safety policy, transform rendering, config schema validation, or backend protocol behavior.
|
||||
|
||||
User-facing command parsing stays in `internal/cli`, including `reconcile-state` flag validation and help text. User-facing config reference stays in `docs/config.md`. External contracts live under `docs/integrations/`.
|
||||
User-facing command parsing stays in `internal/cli`, including `reconcile-state` and `prune` flag validation and help text. User-facing config reference stays in `docs/config.md`. External contracts live under `docs/integrations/`.
|
||||
|
||||
## Config Fields Used
|
||||
|
||||
@@ -38,7 +38,7 @@ Reconcile-state workflows load one configured pipeline/destination selector, ope
|
||||
|
||||
Prune planning consumes a parsed destination state document and a validated retention prune policy, then returns owner-scoped managed output records that would be pruned or preserved. Planning uses output `updated_at` timestamps, applies `keep_latest` before `older_than` when both are configured, and does not open storage, delete files, or rewrite state.
|
||||
|
||||
Prune execution loads one configured pipeline/destination selector, opens that destination root, parses the root `.distributor.json`, and builds a plan from the destination retention policy. Dry-run returns the same planned and preserved managed output records without deleting files or rewriting state. Apply mode deletes only planned managed output paths, never unmanaged files or `.distributor.json`, then removes confirmed deleted records from state and updates the state timestamp. If a delete fails after earlier deletes succeeded, it rewrites state only for the confirmed deletions and preserves records for the failed and unattempted outputs so a retry remains accurate.
|
||||
Prune execution loads one configured pipeline/destination selector, opens that destination root, parses the root `.distributor.json`, and builds a plan from the destination retention policy. Dry-run returns the same planned and preserved managed output records without deleting files or rewriting state. Apply mode deletes only planned managed output paths, never unmanaged files or `.distributor.json`, then removes confirmed deleted records from state and updates the state timestamp. If a delete fails after earlier deletes succeeded, it rewrites state only for the confirmed deletions and preserves records for the failed and unattempted outputs so a retry remains accurate. Text output reports `changed`, `would_change`, or `unchanged`; JSON output uses the shared app envelope.
|
||||
|
||||
HTTP uploads stage and validate archives before enqueueing a pipeline run with a local staged source root. Go producers can use the public `pkg/upload` package to create client-side gzip tar uploads for this server contract; `internal/app` remains the server-side orchestration boundary and does not import that producer package.
|
||||
|
||||
@@ -65,6 +65,7 @@ HTTP upload startup fails if upload tokens are missing, empty, or duplicated. Up
|
||||
- `internal/app/*_test.go`
|
||||
- `internal/app/prune_test.go`
|
||||
- `internal/cli/reconcile_state_test.go`
|
||||
- `internal/cli/prune_test.go`
|
||||
- `internal/cli/root_test.go`
|
||||
- `internal/config/*_test.go`
|
||||
- `internal/ingest/*_test.go`
|
||||
|
||||
@@ -14,7 +14,7 @@ Output from planning is a `Plan` with action, reason, destination identity, sele
|
||||
|
||||
## Boundaries
|
||||
|
||||
The package does not parse CLI flags, load config files, open concrete adapters, discover source bundles, select fixed-path bundle candidates, register transforms, or render command output. The app layer supplies validated request data and concrete dependencies.
|
||||
The package does not parse CLI flags, load config files, open concrete adapters, discover source bundles, select fixed-path bundle candidates, register transforms, prune retained outputs, or render command output. The app layer supplies validated request data and concrete dependencies.
|
||||
|
||||
External destination state semantics are documented in `docs/integrations/destination-state.md`.
|
||||
|
||||
@@ -42,6 +42,8 @@ Shared-root execution writes schema version `3` state. It preserves unrelated ow
|
||||
|
||||
Forced replacement is explicit per request and deletes the bounded destination bundle path before writing new outputs and state.
|
||||
|
||||
Retention pruning is not part of publish execution and does not run automatically after a successful publish. The app-level prune workflow uses destination state after publication to select managed outputs for deletion.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Planning fails for incomplete requests, invalid publish/transform policy, invalid state mode, invalid reconciliation mode, output path collisions, invalid destination state, unmanaged destination content without force, shared-root owner path conflicts, conflict outcomes not allowed by transfer policy, unresolved transforms, invalid Markdown output selection, and invalid link URL planning.
|
||||
|
||||
@@ -50,7 +50,7 @@ Shared-root comparison is owner-scoped. It compares only the owner keyed by the
|
||||
|
||||
Missing-output removal helpers are pure state transformations used by app-level state repair. They remove matching output records only and leave storage inspection, timestamp updates, validation, and state rewrites to callers.
|
||||
|
||||
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution uses the missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
|
||||
Prune planning helpers are pure. They select managed output candidates, sort deterministically by `updated_at` and path, preserve the newest `keep_latest` candidates before evaluating `older_than`, and return planned prune/preserve lists without mutating state. App-level prune execution and the `prune` command use the missing-output removal helpers to remove only confirmed deleted records after storage deletion succeeds.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
|
||||
@@ -116,6 +116,36 @@ Without `--dry-run`, it removes missing managed output records from valid state
|
||||
|
||||
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, repair is scoped to the selected owner by default. Add `--all-owners` only when every owner in the selected shared-root state should have missing managed output records removed.
|
||||
|
||||
## Managed Output Pruning
|
||||
|
||||
Use `prune` when a destination config has `retention.prune.enabled: true` and old managed outputs should be removed according to that configured policy. Pruning is never automatic after publish.
|
||||
|
||||
Preview selected managed outputs first:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id> \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
Apply after reviewing the report:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id> \
|
||||
--apply
|
||||
```
|
||||
|
||||
The command opens the configured destination root selected by `--pipeline` and `--destination`, reads the root `.distributor.json`, and plans from the selected destination's `retention.prune` policy. It uses managed output `updated_at` timestamps. When both `keep_latest` and `older_than` are configured, it preserves the newest `keep_latest` outputs before applying the age policy.
|
||||
|
||||
`--dry-run` does not delete outputs or rewrite state. `--apply` deletes only planned managed output paths, preserves unmanaged files, preserves `.distributor.json`, removes confirmed deleted records from state, and updates the state timestamp. If a delete fails after earlier deletes succeed, state is rewritten only for confirmed deletions; failed and unattempted output records remain so retry remains accurate.
|
||||
|
||||
For single-owner state, the state owner must match the selected pipeline and destination. For shared-root state, pruning is scoped to the selected owner and preserves other owners.
|
||||
|
||||
## Dry Runs And Output Review
|
||||
|
||||
`run --dry-run` loads config, resolves credentials, discovers source bundles, opens destinations, inspects destination state, builds publish plans, and prints actions. It does not write outputs, `.distributor.json`, or SSH `known_hosts` entries. For reconciliation, dry runs report the same high-level action labels as execution; inspect the configured destination's `reconciliation.mode` to determine whether `replace_older` will replace the managed set or merge into it.
|
||||
@@ -254,6 +284,7 @@ Use these recovery boundaries:
|
||||
- For unmanaged destination content, move unrelated files aside or use a different destination path before publishing.
|
||||
- For shared-root ownership conflicts, change one owner so it writes a different destination path, or use a separate destination root.
|
||||
- For missing managed output files recorded in state, run `reconcile-state --dry-run`, then apply `reconcile-state` if the missing files should no longer be considered managed.
|
||||
- For configured retention cleanup, run `prune --dry-run`, then apply `prune --apply` after reviewing the managed output list.
|
||||
- For failed writes, inspect the destination bundle path, remove only confirmed partial outputs if needed, then rerun `--dry-run`. In merge mode, retained outputs may be intentional managed outputs from the prior state.
|
||||
- For state conflicts, verify the source, pipeline, destination, and existing `.distributor.json` before considering `--force`.
|
||||
- For HTTP upload failures, inspect `/runs/<run-id>` while retained; after expiry or restart, rely on destination state and logs/output from the publishing run.
|
||||
|
||||
@@ -87,6 +87,24 @@ Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` valu
|
||||
|
||||
Reference: [CLI](cli.md#reconcile-state).
|
||||
|
||||
## Prune Selector Or Mode Is Missing Or Wrong
|
||||
|
||||
Symptom: `prune requires --config`, `requires --pipeline`, `requires --destination`, `requires exactly one of --dry-run or --apply`, `pipeline "<id>" not found`, `destination <id> not found`, or `state owner is ... not ...`.
|
||||
|
||||
Likely cause: the command did not identify one configured destination root, did not choose exactly one execution mode, or the selected root contains state for a different single-owner pipeline/destination.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
go run ./cmd/distributor prune --help
|
||||
rg -n 'retention:|prune:|pipelines:|destinations:|id:' <config-path>
|
||||
cat <destination-path>/.distributor.json
|
||||
```
|
||||
|
||||
Safe fix: pass the configured `--config`, `--pipeline`, and `--destination` values that identify the destination root containing the state file. Use `--dry-run` for read-only review or `--apply` for deletion, but not both. For unrelated single-owner state, use the correct config selector or a separate destination root; `prune` will not take over mismatched state.
|
||||
|
||||
Reference: [CLI](cli.md#prune).
|
||||
|
||||
## Output Format Is Invalid
|
||||
|
||||
Symptom: `format must be text or json`.
|
||||
@@ -220,6 +238,28 @@ Safe fix: if the missing files should no longer be managed, rerun the same comma
|
||||
|
||||
Reference: [Operations](operations.md#destination-state-repair).
|
||||
|
||||
## Prune Reports No Planned Deletes
|
||||
|
||||
Symptom: `prune --dry-run` reports `planned=0` or JSON `planned_outputs: []`.
|
||||
|
||||
Likely cause: pruning is disabled for the selected destination, every managed output is preserved by `keep_latest`, no managed output is older than `older_than`, or the selected shared-root owner has no eligible outputs.
|
||||
|
||||
Diagnostic:
|
||||
|
||||
```sh
|
||||
rg -n 'retention:|prune:|older_than:|keep_latest:' <config-path>
|
||||
go run ./cmd/distributor prune \
|
||||
--config <config-path> \
|
||||
--pipeline <pipeline-id> \
|
||||
--destination <destination-id> \
|
||||
--dry-run \
|
||||
--format json
|
||||
```
|
||||
|
||||
Safe fix: verify the selected destination's retention policy and the managed output `updated_at` timestamps in `.distributor.json`. Adjust config if the policy is too conservative, then rerun `--dry-run` before using `--apply`.
|
||||
|
||||
Reference: [Operations](operations.md#managed-output-pruning).
|
||||
|
||||
## Destination Is Newer Than Source
|
||||
|
||||
Symptom: `skip_destination_newer` or `destination is newer and replacement requires --force`.
|
||||
|
||||
@@ -3,6 +3,7 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
@@ -16,6 +17,8 @@ type PruneOptions struct {
|
||||
DestinationID string
|
||||
DryRun bool
|
||||
Now time.Time
|
||||
Stdout io.Writer
|
||||
OutputFormat OutputFormat
|
||||
}
|
||||
|
||||
type PrunePlanOptions struct {
|
||||
@@ -63,6 +66,9 @@ type PruneOutputRecord struct {
|
||||
}
|
||||
|
||||
func Prune(ctx context.Context, options PruneOptions) (PruneReport, error) {
|
||||
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
@@ -106,7 +112,14 @@ func pruneSetupWithBackendFactory(ctx context.Context, setup runtimeSetup, optio
|
||||
}
|
||||
defer closeBackend(destinationBackend)
|
||||
|
||||
return executePrune(ctx, destinationBackend, pipeline, destination, options)
|
||||
report, err := executePrune(ctx, destinationBackend, pipeline, destination, options)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if err := WritePruneReport(options.Stdout, options.OutputFormat, report); err != nil {
|
||||
return PruneReport{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func requirePruneScope(options PruneOptions) error {
|
||||
@@ -292,3 +305,35 @@ func pruneOutputRecords(candidates []state.PruneCandidate) []PruneOutputRecord {
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func WritePruneReport(w io.Writer, format OutputFormat, report PruneReport) error {
|
||||
if IsJSONOutput(format) {
|
||||
return WriteJSONEnvelope(w, "prune", true, nil, report, nil)
|
||||
}
|
||||
return writePruneReportText(w, report)
|
||||
}
|
||||
|
||||
func writePruneReportText(w io.Writer, report PruneReport) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
status := "unchanged"
|
||||
if report.StateChanged {
|
||||
status = "changed"
|
||||
} else if report.WouldChange {
|
||||
status = "would_change"
|
||||
}
|
||||
_, err := fmt.Fprintf(w, "Prune: pipeline=%s destination=%s backend=%s root=%s status=%s checked=%d planned=%d deleted=%d preserved=%d dry_run=%t\n",
|
||||
report.PipelineID,
|
||||
report.DestinationID,
|
||||
report.Backend,
|
||||
report.RootPath,
|
||||
status,
|
||||
report.CheckedCount,
|
||||
len(report.PlannedOutputs),
|
||||
len(report.DeletedOutputs),
|
||||
len(report.PreservedOutputs),
|
||||
report.DryRun,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
88
internal/cli/prune.go
Normal file
88
internal/cli/prune.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||
)
|
||||
|
||||
func pruneCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
if hasHelp(args) {
|
||||
printPruneHelp(stdout)
|
||||
return exitOK
|
||||
}
|
||||
|
||||
flags := newFlagSet("prune", stderr)
|
||||
configPath := flags.String("config", "", "path to config file")
|
||||
pipelineID := flags.String("pipeline", "", "pipeline id")
|
||||
destinationID := flags.String("destination", "", "destination id")
|
||||
dryRun := flags.Bool("dry-run", false, "report planned deletes without deleting outputs or rewriting state")
|
||||
apply := flags.Bool("apply", false, "delete planned managed outputs and rewrite state")
|
||||
formatFlag := addFormatFlag(flags)
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if rejectPositionalArgs(stderr, "prune", flags.Args()) {
|
||||
return exitUsage
|
||||
}
|
||||
format, ok := parseOutputFormat(stderr, "prune", *formatFlag)
|
||||
if !ok {
|
||||
return exitUsage
|
||||
}
|
||||
if !validatePruneFlags(stderr, *configPath, *pipelineID, *destinationID, *dryRun, *apply) {
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
if _, err := app.Prune(ctx, app.PruneOptions{
|
||||
ConfigPath: *configPath,
|
||||
PipelineID: *pipelineID,
|
||||
DestinationID: *destinationID,
|
||||
DryRun: *dryRun,
|
||||
Stdout: stdout,
|
||||
OutputFormat: format,
|
||||
}); err != nil {
|
||||
return fail(stderr, err)
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func validatePruneFlags(stderr io.Writer, configPath, pipelineID, destinationID string, dryRun, apply bool) bool {
|
||||
if configPath == "" {
|
||||
fmt.Fprintf(stderr, "%s: prune requires --config\n", app.Name)
|
||||
return false
|
||||
}
|
||||
if pipelineID == "" {
|
||||
fmt.Fprintf(stderr, "%s: prune requires --pipeline\n", app.Name)
|
||||
return false
|
||||
}
|
||||
if destinationID == "" {
|
||||
fmt.Fprintf(stderr, "%s: prune requires --destination\n", app.Name)
|
||||
return false
|
||||
}
|
||||
if dryRun == apply {
|
||||
fmt.Fprintf(stderr, "%s: prune requires exactly one of --dry-run or --apply\n", app.Name)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func printPruneHelp(w io.Writer) {
|
||||
fmt.Fprint(w, `Usage:
|
||||
distributor prune --config <path> --pipeline <id> --destination <id> (--dry-run|--apply) [--format text|json]
|
||||
|
||||
Options:
|
||||
--config <path> Path to config file
|
||||
--pipeline <id> Pipeline id that selects the destination root
|
||||
--destination <id> Destination id that selects the destination root
|
||||
--dry-run Report planned deletes without deleting outputs or rewriting state
|
||||
--apply Delete planned managed outputs and rewrite state
|
||||
--format text|json Output format
|
||||
|
||||
Prune reads the selected destination's configured retention policy and managed
|
||||
state, then plans owner-scoped managed output deletion. --dry-run is read-only.
|
||||
--apply deletes only planned managed output paths, preserves unmanaged files,
|
||||
and rewrites state after confirmed deletes.
|
||||
`)
|
||||
}
|
||||
210
internal/cli/prune_test.go
Normal file
210
internal/cli/prune_test.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestExecutePruneRejectsInvalidFlags(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
wantStderr string
|
||||
}{
|
||||
{
|
||||
name: "missing config",
|
||||
args: []string{"prune", "--pipeline", "reports", "--destination", "archive", "--dry-run"},
|
||||
wantStderr: "requires --config",
|
||||
},
|
||||
{
|
||||
name: "missing pipeline",
|
||||
args: []string{"prune", "--config", "config.yml", "--destination", "archive", "--dry-run"},
|
||||
wantStderr: "requires --pipeline",
|
||||
},
|
||||
{
|
||||
name: "missing destination",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--dry-run"},
|
||||
wantStderr: "requires --destination",
|
||||
},
|
||||
{
|
||||
name: "missing mode",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive"},
|
||||
wantStderr: "requires exactly one of --dry-run or --apply",
|
||||
},
|
||||
{
|
||||
name: "conflicting modes",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "--apply"},
|
||||
wantStderr: "requires exactly one of --dry-run or --apply",
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "--format", "xml"},
|
||||
wantStderr: "format must be text or json",
|
||||
},
|
||||
{
|
||||
name: "positional",
|
||||
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "extra"},
|
||||
wantStderr: "does not accept positional arguments",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := Execute(context.Background(), tt.args, &stdout, &stderr)
|
||||
if code != exitUsage {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitUsage, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tt.wantStderr) {
|
||||
t.Fatalf("stderr = %q, want substring %q", stderr.String(), tt.wantStderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePruneDryRunReportsWithoutWriting(t *testing.T) {
|
||||
destinationRoot, configPath := writePruneLocalFixture(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"prune",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
"--dry-run",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if output := stdout.String(); !strings.Contains(output, "status=would_change") || !strings.Contains(output, "planned=2") || !strings.Contains(output, "deleted=0") {
|
||||
t.Fatalf("stdout = %q, want dry-run prune summary", output)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePruneJSONReport(t *testing.T) {
|
||||
_, configPath := writePruneLocalFixture(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"prune",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
"--dry-run",
|
||||
"--format", "json",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
envelope := decodeEnvelope(t, &stdout)
|
||||
if envelope["command"] != "prune" || envelope["ok"] != true {
|
||||
t.Fatalf("envelope = %#v, want prune ok", envelope)
|
||||
}
|
||||
result := envelopeResult(t, envelope)
|
||||
if result["would_change"] != true || result["state_changed"] != false || result["dry_run"] != true {
|
||||
t.Fatalf("result = %#v, want dry-run pending change", result)
|
||||
}
|
||||
planned, ok := result["planned_outputs"].([]any)
|
||||
if !ok || len(planned) != 2 {
|
||||
t.Fatalf("planned outputs = %#v, want two", result["planned_outputs"])
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePruneApplyDeletesManagedOutputs(t *testing.T) {
|
||||
destinationRoot, configPath := writePruneLocalFixture(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
|
||||
code := Execute(context.Background(), []string{
|
||||
"prune",
|
||||
"--config", configPath,
|
||||
"--pipeline", "reports",
|
||||
"--destination", "archive",
|
||||
"--apply",
|
||||
}, &stdout, &stderr)
|
||||
|
||||
if code != exitOK {
|
||||
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||
}
|
||||
if output := stdout.String(); !strings.Contains(output, "status=changed") || !strings.Contains(output, "deleted=2") {
|
||||
t.Fatalf("stdout = %q, want applied prune summary", output)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) {
|
||||
t.Fatalf("report.md stat error = %v, want not exist", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, "summary.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("summary.txt stat error = %v, want not exist", err)
|
||||
}
|
||||
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
|
||||
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
|
||||
t.Fatalf("state file stat error = %v", err)
|
||||
}
|
||||
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
|
||||
if got := state.ManagedOutputPaths(destinationState); len(got) != 0 {
|
||||
t.Fatalf("state outputs = %#v, want none", got)
|
||||
}
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func writePruneLocalFixture(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
|
||||
for _, file := range testutil.DefaultSourceFiles() {
|
||||
path := filepath.Join(destinationRoot, filepath.FromSlash(file.Path))
|
||||
if err := os.WriteFile(path, []byte(file.Data), 0o600); err != nil {
|
||||
t.Fatalf("write destination output: %v", err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
|
||||
t.Fatalf("write unmanaged output: %v", err)
|
||||
}
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
config := `
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: ` + sourceRoot + `
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: ` + destinationRoot + `
|
||||
retention:
|
||||
prune:
|
||||
enabled: true
|
||||
older_than: 1h
|
||||
`
|
||||
if err := os.WriteFile(configPath, []byte(strings.TrimSpace(config)+"\n"), 0o600); err != nil {
|
||||
t.Fatalf("write prune config: %v", err)
|
||||
}
|
||||
return destinationRoot, configPath
|
||||
}
|
||||
@@ -31,6 +31,8 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
return runCommand(ctx, args[1:], stdout, stderr)
|
||||
case "reconcile-state":
|
||||
return reconcileStateCommand(ctx, args[1:], stdout, stderr)
|
||||
case "prune":
|
||||
return pruneCommand(ctx, args[1:], stdout, stderr)
|
||||
case "serve":
|
||||
return serveCommand(ctx, args[1:], stdout, stderr)
|
||||
case "validate":
|
||||
@@ -57,6 +59,7 @@ Commands:
|
||||
run Run configured distribution pipelines
|
||||
reconcile-state
|
||||
Repair missing managed-output records in destination state
|
||||
prune Prune managed outputs using configured retention policy
|
||||
serve Run the HTTP upload server
|
||||
validate Validate a source bundle or bundle tree
|
||||
inspect Inspect bundles or distributor state
|
||||
|
||||
Reference in New Issue
Block a user