eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Expose managed output pruning

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 19:38:49 +00:00
parent 6daddad543
commit ea562c1c3a
12 changed files with 481 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
internal/app/prune.go
View File

@@ -3,6 +3,7 @@ package app
import (
"context"
"fmt"
"io"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/config"
@@ -16,6 +17,8 @@ type PruneOptions struct {
DestinationID string
DryRun bool
Now time.Time
Stdout io.Writer
OutputFormat OutputFormat
}
type PrunePlanOptions struct {
@@ -63,6 +66,9 @@ type PruneOutputRecord struct {
}
func Prune(ctx context.Context, options PruneOptions) (PruneReport, error) {
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
return PruneReport{}, err
}
if err := ctx.Err(); err != nil {
return PruneReport{}, err
}
@@ -106,7 +112,14 @@ func pruneSetupWithBackendFactory(ctx context.Context, setup runtimeSetup, optio
}
defer closeBackend(destinationBackend)
return executePrune(ctx, destinationBackend, pipeline, destination, options)
report, err := executePrune(ctx, destinationBackend, pipeline, destination, options)
if err != nil {
return report, err
}
if err := WritePruneReport(options.Stdout, options.OutputFormat, report); err != nil {
return PruneReport{}, err
}
return report, nil
}
func requirePruneScope(options PruneOptions) error {
@@ -292,3 +305,35 @@ func pruneOutputRecords(candidates []state.PruneCandidate) []PruneOutputRecord {
}
return records
}
func WritePruneReport(w io.Writer, format OutputFormat, report PruneReport) error {
if IsJSONOutput(format) {
return WriteJSONEnvelope(w, "prune", true, nil, report, nil)
}
return writePruneReportText(w, report)
}
func writePruneReportText(w io.Writer, report PruneReport) error {
if w == nil {
return nil
}
status := "unchanged"
if report.StateChanged {
status = "changed"
} else if report.WouldChange {
status = "would_change"
}
_, err := fmt.Fprintf(w, "Prune: pipeline=%s destination=%s backend=%s root=%s status=%s checked=%d planned=%d deleted=%d preserved=%d dry_run=%t\n",
report.PipelineID,
report.DestinationID,
report.Backend,
report.RootPath,
status,
report.CheckedCount,
len(report.PlannedOutputs),
len(report.DeletedOutputs),
len(report.PreservedOutputs),
report.DryRun,
)
return err
}

88
internal/cli/prune.go Normal file
View File

@@ -0,0 +1,88 @@
package cli
import (
"context"
"fmt"
"io"
"gitea.maximumdirect.net/eric/distributor/internal/app"
)
func pruneCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
if hasHelp(args) {
printPruneHelp(stdout)
return exitOK
}
flags := newFlagSet("prune", stderr)
configPath := flags.String("config", "", "path to config file")
pipelineID := flags.String("pipeline", "", "pipeline id")
destinationID := flags.String("destination", "", "destination id")
dryRun := flags.Bool("dry-run", false, "report planned deletes without deleting outputs or rewriting state")
apply := flags.Bool("apply", false, "delete planned managed outputs and rewrite state")
formatFlag := addFormatFlag(flags)
if err := flags.Parse(args); err != nil {
return exitUsage
}
if rejectPositionalArgs(stderr, "prune", flags.Args()) {
return exitUsage
}
format, ok := parseOutputFormat(stderr, "prune", *formatFlag)
if !ok {
return exitUsage
}
if !validatePruneFlags(stderr, *configPath, *pipelineID, *destinationID, *dryRun, *apply) {
return exitUsage
}
if _, err := app.Prune(ctx, app.PruneOptions{
ConfigPath: *configPath,
PipelineID: *pipelineID,
DestinationID: *destinationID,
DryRun: *dryRun,
Stdout: stdout,
OutputFormat: format,
}); err != nil {
return fail(stderr, err)
}
return exitOK
}
func validatePruneFlags(stderr io.Writer, configPath, pipelineID, destinationID string, dryRun, apply bool) bool {
if configPath == "" {
fmt.Fprintf(stderr, "%s: prune requires --config\n", app.Name)
return false
}
if pipelineID == "" {
fmt.Fprintf(stderr, "%s: prune requires --pipeline\n", app.Name)
return false
}
if destinationID == "" {
fmt.Fprintf(stderr, "%s: prune requires --destination\n", app.Name)
return false
}
if dryRun == apply {
fmt.Fprintf(stderr, "%s: prune requires exactly one of --dry-run or --apply\n", app.Name)
return false
}
return true
}
func printPruneHelp(w io.Writer) {
fmt.Fprint(w, `Usage:
distributor prune --config <path> --pipeline <id> --destination <id> (--dry-run|--apply) [--format text|json]
Options:
--config <path> Path to config file
--pipeline <id> Pipeline id that selects the destination root
--destination <id> Destination id that selects the destination root
--dry-run Report planned deletes without deleting outputs or rewriting state
--apply Delete planned managed outputs and rewrite state
--format text|json Output format
Prune reads the selected destination's configured retention policy and managed
state, then plans owner-scoped managed output deletion. --dry-run is read-only.
--apply deletes only planned managed output paths, preserves unmanaged files,
and rewrites state after confirmed deletes.
`)
}

210
internal/cli/prune_test.go Normal file
View File

@@ -0,0 +1,210 @@
package cli
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
func TestExecutePruneRejectsInvalidFlags(t *testing.T) {
tests := []struct {
name string
args []string
wantStderr string
}{
{
name: "missing config",
args: []string{"prune", "--pipeline", "reports", "--destination", "archive", "--dry-run"},
wantStderr: "requires --config",
},
{
name: "missing pipeline",
args: []string{"prune", "--config", "config.yml", "--destination", "archive", "--dry-run"},
wantStderr: "requires --pipeline",
},
{
name: "missing destination",
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--dry-run"},
wantStderr: "requires --destination",
},
{
name: "missing mode",
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive"},
wantStderr: "requires exactly one of --dry-run or --apply",
},
{
name: "conflicting modes",
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "--apply"},
wantStderr: "requires exactly one of --dry-run or --apply",
},
{
name: "invalid format",
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "--format", "xml"},
wantStderr: "format must be text or json",
},
{
name: "positional",
args: []string{"prune", "--config", "config.yml", "--pipeline", "reports", "--destination", "archive", "--dry-run", "extra"},
wantStderr: "does not accept positional arguments",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
code := Execute(context.Background(), tt.args, &stdout, &stderr)
if code != exitUsage {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitUsage, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("stdout = %q, want empty", stdout.String())
}
if !strings.Contains(stderr.String(), tt.wantStderr) {
t.Fatalf("stderr = %q, want substring %q", stderr.String(), tt.wantStderr)
}
})
}
}
func TestExecutePruneDryRunReportsWithoutWriting(t *testing.T) {
destinationRoot, configPath := writePruneLocalFixture(t)
var stdout, stderr bytes.Buffer
code := Execute(context.Background(), []string{
"prune",
"--config", configPath,
"--pipeline", "reports",
"--destination", "archive",
"--dry-run",
}, &stdout, &stderr)
if code != exitOK {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
}
if output := stdout.String(); !strings.Contains(output, "status=would_change") || !strings.Contains(output, "planned=2") || !strings.Contains(output, "deleted=0") {
t.Fatalf("stdout = %q, want dry-run prune summary", output)
}
assertLocalFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
assertLocalFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if stderr.Len() != 0 {
t.Fatalf("stderr = %q, want empty", stderr.String())
}
}
func TestExecutePruneJSONReport(t *testing.T) {
_, configPath := writePruneLocalFixture(t)
var stdout, stderr bytes.Buffer
code := Execute(context.Background(), []string{
"prune",
"--config", configPath,
"--pipeline", "reports",
"--destination", "archive",
"--dry-run",
"--format", "json",
}, &stdout, &stderr)
if code != exitOK {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
}
envelope := decodeEnvelope(t, &stdout)
if envelope["command"] != "prune" || envelope["ok"] != true {
t.Fatalf("envelope = %#v, want prune ok", envelope)
}
result := envelopeResult(t, envelope)
if result["would_change"] != true || result["state_changed"] != false || result["dry_run"] != true {
t.Fatalf("result = %#v, want dry-run pending change", result)
}
planned, ok := result["planned_outputs"].([]any)
if !ok || len(planned) != 2 {
t.Fatalf("planned outputs = %#v, want two", result["planned_outputs"])
}
if stderr.Len() != 0 {
t.Fatalf("stderr = %q, want empty", stderr.String())
}
}
func TestExecutePruneApplyDeletesManagedOutputs(t *testing.T) {
destinationRoot, configPath := writePruneLocalFixture(t)
var stdout, stderr bytes.Buffer
code := Execute(context.Background(), []string{
"prune",
"--config", configPath,
"--pipeline", "reports",
"--destination", "archive",
"--apply",
}, &stdout, &stderr)
if code != exitOK {
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
}
if output := stdout.String(); !strings.Contains(output, "status=changed") || !strings.Contains(output, "deleted=2") {
t.Fatalf("stdout = %q, want applied prune summary", output)
}
if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) {
t.Fatalf("report.md stat error = %v, want not exist", err)
}
if _, err := os.Stat(filepath.Join(destinationRoot, "summary.txt")); !os.IsNotExist(err) {
t.Fatalf("summary.txt stat error = %v, want not exist", err)
}
assertLocalFile(t, filepath.Join(destinationRoot, "extra.txt"), "unmanaged")
if _, err := os.Stat(filepath.Join(destinationRoot, storage.StateFileName)); err != nil {
t.Fatalf("state file stat error = %v", err)
}
destinationState := testutil.ReadDestinationState(t, filepath.Join(destinationRoot, storage.StateFileName))
if got := state.ManagedOutputPaths(destinationState); len(got) != 0 {
t.Fatalf("state outputs = %#v, want none", got)
}
if stderr.Len() != 0 {
t.Fatalf("stderr = %q, want empty", stderr.String())
}
}
func writePruneLocalFixture(t *testing.T) (string, string) {
t.Helper()
sourceRoot := t.TempDir()
destinationRoot := t.TempDir()
manifest := testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
testutil.WriteDestinationState(t, destinationRoot, "", manifest, testutil.DestinationStateOptions{})
for _, file := range testutil.DefaultSourceFiles() {
path := filepath.Join(destinationRoot, filepath.FromSlash(file.Path))
if err := os.WriteFile(path, []byte(file.Data), 0o600); err != nil {
t.Fatalf("write destination output: %v", err)
}
}
if err := os.WriteFile(filepath.Join(destinationRoot, "extra.txt"), []byte("unmanaged"), 0o600); err != nil {
t.Fatalf("write unmanaged output: %v", err)
}
configPath := filepath.Join(t.TempDir(), "config.yml")
config := `
pipelines:
- id: reports
source:
backend: local
path: ` + sourceRoot + `
destinations:
- id: archive
backend: local
path: ` + destinationRoot + `
retention:
prune:
enabled: true
older_than: 1h
`
if err := os.WriteFile(configPath, []byte(strings.TrimSpace(config)+"\n"), 0o600); err != nil {
t.Fatalf("write prune config: %v", err)
}
return destinationRoot, configPath
}

3
internal/cli/root.go
View File

@@ -31,6 +31,8 @@ func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
return runCommand(ctx, args[1:], stdout, stderr)
case "reconcile-state":
return reconcileStateCommand(ctx, args[1:], stdout, stderr)
case "prune":
return pruneCommand(ctx, args[1:], stdout, stderr)
case "serve":
return serveCommand(ctx, args[1:], stdout, stderr)
case "validate":
@@ -57,6 +59,7 @@ Commands:
run Run configured distribution pipelines
reconcile-state
Repair missing managed-output records in destination state
prune Prune managed outputs using configured retention policy
serve Run the HTTP upload server
validate Validate a source bundle or bundle tree
inspect Inspect bundles or distributor state