eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Support catalog maintenance commands

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-19 15:58:36 +00:00
parent 91b72478d5
commit cf7e62733e
8 changed files with 295 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
internal/app/prune_test.go
View File

@@ -15,7 +15,7 @@ import (
)
func TestPlanPruneDisabledPolicy(t *testing.T) {
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
document := state.StateDocument{Catalog: &state.CatalogState{}}
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
PipelineID: "reports",
DestinationID: "archive",
@@ -28,12 +28,12 @@ func TestPlanPruneDisabledPolicy(t *testing.T) {
}
}
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
func TestPlanPruneCatalogOutputs(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
olderThan := config.Duration(48 * time.Hour)
destinationState := pruneSingleOwnerState(now)
catalog := pruneCatalogState(now)
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
Enabled: true,
OlderThan: &olderThan,
}, PrunePlanOptions{
@@ -52,12 +52,12 @@ func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
}
}
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
func TestPlanPruneCatalogCurrentOwnerOnly(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
keepLatest := 0
sharedRoot := pruneSharedRootState(now)
catalog := pruneCatalogState(now)
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
Enabled: true,
KeepLatest: &keepLatest,
}, PrunePlanOptions{
@@ -68,21 +68,20 @@ func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
if err != nil {
t.Fatalf("PlanPrune() error = %v", err)
}
if got, want := report.CheckedCount, 1; got != want {
if got, want := report.CheckedCount, 2; got != want {
t.Fatalf("checked count = %d, want %d", got, want)
}
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt,fresh.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
}
func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *testing.T) {
t.Skip("catalog prune execution is covered by the catalog maintenance work")
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
original := pruneSingleOwnerState(now)
writeFakeSingleOwnerStateForPrune(t, backend, original)
original := pruneCatalogState(now)
writeFakeCatalogState(t, backend, original)
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -103,21 +102,20 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "old.txt,fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want original outputs", got)
}
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
if !catalog.UpdatedAt.Equal(original.UpdatedAt) {
t.Fatalf("state updated_at = %s, want original %s", catalog.UpdatedAt, original.UpdatedAt)
}
}
func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
t.Skip("catalog prune execution is covered by the catalog maintenance work")
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -136,24 +134,27 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
}
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
assertFakeStateExists(t, backend)
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want fresh.txt", got)
}
if !destinationState.UpdatedAt.Equal(now) {
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
if catalog.SchemaVersion != state.CatalogSchemaVersion {
t.Fatalf("state schema_version = %d, want %d", catalog.SchemaVersion, state.CatalogSchemaVersion)
}
if !catalog.UpdatedAt.Equal(now) {
t.Fatalf("state updated_at = %s, want %s", catalog.UpdatedAt, now)
}
}
func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
t.Skip("catalog prune execution is covered by the catalog maintenance work")
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
keepLatest := 0
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -173,19 +174,17 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
assertFakeStateExists(t, backend)
destinationState := readFakeSingleOwnerState(t, backend)
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("state outputs = %q, want only failed output preserved", got)
}
}
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
t.Skip("catalog prune execution is covered by the catalog maintenance work")
func TestPrunePreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
backend := fake.New()
keepLatest := 0
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
writeFakeCatalogState(t, backend, pruneCatalogState(now))
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
@@ -196,93 +195,56 @@ func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.
if err != nil {
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
}
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
t.Fatalf("deleted outputs = %q, want %q", got, want)
}
testutil.AssertFakeMissing(t, backend, "archive.txt")
testutil.AssertFakeFile(t, backend, "html.txt", "old")
testutil.AssertFakeMissing(t, backend, "old.txt")
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
sharedRoot := readFakeSharedRootStateForApp(t, backend)
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
catalog := readFakeCatalogState(t, backend)
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
}
}
func pruneSingleOwnerState(now time.Time) state.DistributorState {
func pruneCatalogState(now time.Time) state.CatalogState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
publishedAt := now.Add(-96 * time.Hour)
return state.DistributorState{
SchemaVersion: state.SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
createdAt := now.Add(-96 * time.Hour)
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
return state.CatalogState{
SchemaVersion: state.CatalogSchemaVersion,
DistributorVersion: "test",
Outputs: []state.OutputFile{{
Path: "old.txt",
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}, {
Path: "fresh.txt",
Kind: state.OutputKindSource,
SourcePath: "summary.txt",
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
CreatedAt: now.Add(-24 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
}},
}
}
func pruneSharedRootState(now time.Time) state.SharedRootState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
archive := state.CurrentOwnerScope("reports", "archive")
html := state.CurrentOwnerScope("reports", "html")
return state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: "test",
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{{
Scope: archive,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
}, {
Scope: html,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
}},
Outputs: []state.SharedRootOutputFile{{
Path: "archive.txt",
CreatedAt: createdAt,
UpdatedAt: createdAt,
State: state.StatePolicy{Mode: state.StateModeCatalog},
Outputs: []state.CatalogOutputFile{{
Path: "old.txt",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
Owner: archive,
SourceID: manifest.ID,
SourceDigest: manifest.Digest,
SourceCreated: manifest.Created,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}, {
Path: "html.txt",
Path: "fresh.txt",
PipelineID: "reports",
DestinationID: "archive",
Source: source,
Kind: state.OutputKindSource,
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: now.Add(-24 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
}, {
Path: "html.txt",
PipelineID: "reports",
DestinationID: "html",
Source: source,
Kind: state.OutputKindSource,
SourcePath: "summary.txt",
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
Owner: html,
SourceID: manifest.ID,
SourceDigest: manifest.Digest,
SourceCreated: manifest.Created,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}},
@@ -323,18 +285,31 @@ func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
}
}
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
func writeFakeCatalogState(t *testing.T, backend *fake.Backend, catalog state.CatalogState) {
t.Helper()
data, err := json.MarshalIndent(destinationState, "", " ")
data, err := json.MarshalIndent(catalog, "", " ")
if err != nil {
t.Fatalf("marshal single-owner state: %v", err)
t.Fatalf("marshal catalog state: %v", err)
}
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
for _, output := range destinationState.Outputs {
for _, output := range catalog.Outputs {
testutil.WriteFakeFile(t, backend, output.Path, "managed")
}
}
func readFakeCatalogState(t *testing.T, backend *fake.Backend) state.CatalogState {
t.Helper()
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
if err != nil {
t.Fatalf("read catalog state: %v", err)
}
catalog, err := state.ParseCatalog(data)
if err != nil {
t.Fatalf("parse catalog state: %v", err)
}
return catalog
}
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
t.Helper()
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {