333 lines
12 KiB
Go
333 lines
12 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
|
)
|
|
|
|
func TestPlanPruneDisabledPolicy(t *testing.T) {
|
|
document := state.StateDocument{Catalog: &state.CatalogState{}}
|
|
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("PlanPrune() error = %v", err)
|
|
}
|
|
if report.Enabled || report.CheckedCount != 0 || len(report.PrunedOutputs) != 0 {
|
|
t.Fatalf("report = %#v, want disabled empty plan", report)
|
|
}
|
|
}
|
|
|
|
func TestPlanPruneCatalogOutputs(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
olderThan := config.Duration(48 * time.Hour)
|
|
catalog := pruneCatalogState(now)
|
|
|
|
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
|
|
Enabled: true,
|
|
OlderThan: &olderThan,
|
|
}, PrunePlanOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("PlanPrune() error = %v", err)
|
|
}
|
|
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("pruned = %q, want %q", got, want)
|
|
}
|
|
if got, want := pruneRecordPaths(report.PreservedOutputs), "fresh.txt"; got != want {
|
|
t.Fatalf("preserved = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestPlanPruneCatalogCurrentOwnerOnly(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
keepLatest := 0
|
|
catalog := pruneCatalogState(now)
|
|
|
|
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
|
|
Enabled: true,
|
|
KeepLatest: &keepLatest,
|
|
}, PrunePlanOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("PlanPrune() error = %v", err)
|
|
}
|
|
if got, want := report.CheckedCount, 2; got != want {
|
|
t.Fatalf("checked count = %d, want %d", got, want)
|
|
}
|
|
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt,fresh.txt"; got != want {
|
|
t.Fatalf("pruned = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
backend := fake.New()
|
|
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
|
original := pruneCatalogState(now)
|
|
writeFakeCatalogState(t, backend, original)
|
|
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
|
|
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
DryRun: true,
|
|
Now: now,
|
|
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
|
if err != nil {
|
|
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
|
}
|
|
if !report.WouldChange || report.StateChanged || len(report.DeletedOutputs) != 0 {
|
|
t.Fatalf("report would_change=%t state_changed=%t deleted=%d, want dry-run only", report.WouldChange, report.StateChanged, len(report.DeletedOutputs))
|
|
}
|
|
if got, want := pruneRecordPaths(report.PlannedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("planned outputs = %q, want %q", got, want)
|
|
}
|
|
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
|
|
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
|
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
catalog := readFakeCatalogState(t, backend)
|
|
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "old.txt,fresh.txt,html.txt" {
|
|
t.Fatalf("state outputs = %q, want original outputs", got)
|
|
}
|
|
if !catalog.UpdatedAt.Equal(original.UpdatedAt) {
|
|
t.Fatalf("state updated_at = %s, want original %s", catalog.UpdatedAt, original.UpdatedAt)
|
|
}
|
|
}
|
|
|
|
func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
backend := fake.New()
|
|
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
|
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
|
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
|
|
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
|
if err != nil {
|
|
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
|
}
|
|
if !report.StateChanged || report.WouldChange {
|
|
t.Fatalf("report state_changed=%t would_change=%t, want applied change", report.StateChanged, report.WouldChange)
|
|
}
|
|
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
|
}
|
|
testutil.AssertFakeMissing(t, backend, "old.txt")
|
|
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
|
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
|
|
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
assertFakeStateExists(t, backend)
|
|
catalog := readFakeCatalogState(t, backend)
|
|
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
|
t.Fatalf("state outputs = %q, want fresh.txt", got)
|
|
}
|
|
if catalog.SchemaVersion != state.CatalogSchemaVersion {
|
|
t.Fatalf("state schema_version = %d, want %d", catalog.SchemaVersion, state.CatalogSchemaVersion)
|
|
}
|
|
if !catalog.UpdatedAt.Equal(now) {
|
|
t.Fatalf("state updated_at = %s, want %s", catalog.UpdatedAt, now)
|
|
}
|
|
}
|
|
|
|
func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
backend := fake.New()
|
|
keepLatest := 0
|
|
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
|
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
|
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
|
|
|
|
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": failingBackend}))
|
|
if err == nil {
|
|
t.Fatal("pruneConfigWithBackendFactory() error = nil, want delete failure")
|
|
}
|
|
if report.FailedOutput == nil || report.FailedOutput.Path != "fresh.txt" {
|
|
t.Fatalf("failed output = %#v, want fresh.txt", report.FailedOutput)
|
|
}
|
|
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
|
}
|
|
testutil.AssertFakeMissing(t, backend, "old.txt")
|
|
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
|
assertFakeStateExists(t, backend)
|
|
catalog := readFakeCatalogState(t, backend)
|
|
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
|
t.Fatalf("state outputs = %q, want only failed output preserved", got)
|
|
}
|
|
}
|
|
|
|
func TestPrunePreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
backend := fake.New()
|
|
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
|
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
|
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
|
|
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
|
if err != nil {
|
|
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
|
}
|
|
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
|
}
|
|
testutil.AssertFakeMissing(t, backend, "old.txt")
|
|
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
|
|
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
catalog := readFakeCatalogState(t, backend)
|
|
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
|
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
|
|
}
|
|
}
|
|
|
|
func pruneCatalogState(now time.Time) state.CatalogState {
|
|
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
|
createdAt := now.Add(-96 * time.Hour)
|
|
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
|
|
return state.CatalogState{
|
|
SchemaVersion: state.CatalogSchemaVersion,
|
|
DistributorVersion: "test",
|
|
CreatedAt: createdAt,
|
|
UpdatedAt: createdAt,
|
|
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
|
Outputs: []state.CatalogOutputFile{{
|
|
Path: "old.txt",
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Source: source,
|
|
Kind: state.OutputKindSource,
|
|
SHA256: manifest.Files[0].SHA256,
|
|
Size: manifest.Files[0].Size,
|
|
CreatedAt: now.Add(-96 * time.Hour),
|
|
UpdatedAt: now.Add(-72 * time.Hour),
|
|
}, {
|
|
Path: "fresh.txt",
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Source: source,
|
|
Kind: state.OutputKindSource,
|
|
SHA256: manifest.Files[0].SHA256,
|
|
Size: manifest.Files[0].Size,
|
|
CreatedAt: now.Add(-24 * time.Hour),
|
|
UpdatedAt: now.Add(-24 * time.Hour),
|
|
}, {
|
|
Path: "html.txt",
|
|
PipelineID: "reports",
|
|
DestinationID: "html",
|
|
Source: source,
|
|
Kind: state.OutputKindSource,
|
|
SHA256: manifest.Files[1].SHA256,
|
|
Size: manifest.Files[1].Size,
|
|
CreatedAt: now.Add(-96 * time.Hour),
|
|
UpdatedAt: now.Add(-72 * time.Hour),
|
|
}},
|
|
}
|
|
}
|
|
|
|
func pruneRecordPaths(records []PruneOutputRecord) string {
|
|
paths := make([]string, 0, len(records))
|
|
for _, record := range records {
|
|
paths = append(paths, record.Path)
|
|
}
|
|
return strings.Join(paths, ",")
|
|
}
|
|
|
|
func pruneS3Config(t *testing.T, policy config.PrunePolicy) config.Config {
|
|
t.Helper()
|
|
cfg := config.Config{Pipelines: []config.Pipeline{{
|
|
ID: "reports",
|
|
Source: config.Backend{Backend: config.BackendLocal, Path: t.TempDir()},
|
|
Destinations: []config.Destination{{
|
|
ID: "archive",
|
|
Backend: config.BackendS3,
|
|
Bucket: "reports",
|
|
Retention: config.RetentionPolicy{
|
|
Prune: policy,
|
|
},
|
|
}},
|
|
}}}
|
|
config.ApplyDefaults(&cfg)
|
|
return cfg
|
|
}
|
|
|
|
func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
|
|
value := config.Duration(duration)
|
|
return config.PrunePolicy{
|
|
Enabled: true,
|
|
OlderThan: &value,
|
|
}
|
|
}
|
|
|
|
func writeFakeCatalogState(t *testing.T, backend *fake.Backend, catalog state.CatalogState) {
|
|
t.Helper()
|
|
data, err := json.MarshalIndent(catalog, "", " ")
|
|
if err != nil {
|
|
t.Fatalf("marshal catalog state: %v", err)
|
|
}
|
|
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
|
for _, output := range catalog.Outputs {
|
|
testutil.WriteFakeFile(t, backend, output.Path, "managed")
|
|
}
|
|
}
|
|
|
|
func readFakeCatalogState(t *testing.T, backend *fake.Backend) state.CatalogState {
|
|
t.Helper()
|
|
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
|
if err != nil {
|
|
t.Fatalf("read catalog state: %v", err)
|
|
}
|
|
catalog, err := state.ParseCatalog(data)
|
|
if err != nil {
|
|
t.Fatalf("parse catalog state: %v", err)
|
|
}
|
|
return catalog
|
|
}
|
|
|
|
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
|
|
t.Helper()
|
|
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {
|
|
t.Fatalf("state file stat error = %v", err)
|
|
}
|
|
}
|
|
|
|
type failingDeleteBackend struct {
|
|
storage.Backend
|
|
failPath string
|
|
}
|
|
|
|
func (b failingDeleteBackend) DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
|
for _, path := range managedOutputPaths {
|
|
if path == b.failPath {
|
|
return storage.NewError(storage.OpDeleteManagedOutputs, "fake", path, storage.ErrPermission, nil)
|
|
}
|
|
}
|
|
return b.Backend.DeleteManagedOutputs(ctx, bundlePath, managedOutputPaths, opts)
|
|
}
|