eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add managed output pruning

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-08 19:32:57 +00:00
parent c67ecf86a9
commit 6daddad543
5 changed files with 392 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
internal/app/prune_test.go
View File

@@ -1,12 +1,16 @@
package app
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
@@ -72,6 +76,134 @@ func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
}
}
func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
original := pruneSingleOwnerState(now)
writeFakeSingleOwnerStateForPrune(t, backend, original)
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
PipelineID: "reports",
DestinationID: "archive",
DryRun: true,
Now: now,
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
if err != nil {
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
}
if !report.WouldChange || report.StateChanged || len(report.DeletedOutputs) != 0 {
t.Fatalf("report would_change=%t state_changed=%t deleted=%d, want dry-run only", report.WouldChange, report.StateChanged, len(report.DeletedOutputs))
}
if got, want := pruneRecordPaths(report.PlannedOutputs), "old.txt"; got != want {
t.Fatalf("planned outputs = %q, want %q", got, want)
}
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
}
}
func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
PipelineID: "reports",
DestinationID: "archive",
Now: now,
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
if err != nil {
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
}
if !report.StateChanged || report.WouldChange {
t.Fatalf("report state_changed=%t would_change=%t, want applied change", report.StateChanged, report.WouldChange)
}
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
t.Fatalf("deleted outputs = %q, want %q", got, want)
}
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
assertFakeStateExists(t, backend)
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
t.Fatalf("state outputs = %q, want fresh.txt", got)
}
if !destinationState.UpdatedAt.Equal(now) {
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
}
}
func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
keepLatest := 0
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
PipelineID: "reports",
DestinationID: "archive",
Now: now,
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": failingBackend}))
if err == nil {
t.Fatal("pruneConfigWithBackendFactory() error = nil, want delete failure")
}
if report.FailedOutput == nil || report.FailedOutput.Path != "fresh.txt" {
t.Fatalf("failed output = %#v, want fresh.txt", report.FailedOutput)
}
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
t.Fatalf("deleted outputs = %q, want %q", got, want)
}
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
assertFakeStateExists(t, backend)
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
t.Fatalf("state outputs = %q, want only failed output preserved", got)
}
}
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
keepLatest := 0
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
PipelineID: "reports",
DestinationID: "archive",
Now: now,
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
if err != nil {
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
}
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
t.Fatalf("deleted outputs = %q, want %q", got, want)
}
testutil.AssertFakeMissing(t, backend, "archive.txt")
testutil.AssertFakeFile(t, backend, "html.txt", "old")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
sharedRoot := readFakeSharedRootStateForApp(t, backend)
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
}
}
func pruneSingleOwnerState(now time.Time) state.DistributorState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
publishedAt := now.Add(-96 * time.Hour)
@@ -160,3 +292,62 @@ func pruneRecordPaths(records []PruneOutputRecord) string {
}
return strings.Join(paths, ",")
}
func pruneS3Config(t *testing.T, policy config.PrunePolicy) config.Config {
t.Helper()
cfg := config.Config{Pipelines: []config.Pipeline{{
ID: "reports",
Source: config.Backend{Backend: config.BackendLocal, Path: t.TempDir()},
Destinations: []config.Destination{{
ID: "archive",
Backend: config.BackendS3,
Bucket: "reports",
Retention: config.RetentionPolicy{
Prune: policy,
},
}},
}}}
config.ApplyDefaults(&cfg)
return cfg
}
func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
value := config.Duration(duration)
return config.PrunePolicy{
Enabled: true,
OlderThan: &value,
}
}
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
t.Helper()
data, err := json.MarshalIndent(destinationState, "", " ")
if err != nil {
t.Fatalf("marshal single-owner state: %v", err)
}
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
for _, output := range destinationState.Outputs {
testutil.WriteFakeFile(t, backend, output.Path, "managed")
}
}
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
t.Helper()
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {
t.Fatalf("state file stat error = %v", err)
}
}
type failingDeleteBackend struct {
storage.Backend
failPath string
}
func (b failingDeleteBackend) DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
for _, path := range managedOutputPaths {
if path == b.failPath {
return storage.NewError(storage.OpDeleteManagedOutputs, "fake", path, storage.ErrPermission, nil)
}
}
return b.Backend.DeleteManagedOutputs(ctx, bundlePath, managedOutputPaths, opts)
}