354 lines
13 KiB
Go
354 lines
13 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
|
)
|
|
|
|
func TestPlanPruneDisabledPolicy(t *testing.T) {
|
|
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
|
|
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("PlanPrune() error = %v", err)
|
|
}
|
|
if report.Enabled || report.CheckedCount != 0 || len(report.PrunedOutputs) != 0 {
|
|
t.Fatalf("report = %#v, want disabled empty plan", report)
|
|
}
|
|
}
|
|
|
|
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
olderThan := config.Duration(48 * time.Hour)
|
|
destinationState := pruneSingleOwnerState(now)
|
|
|
|
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
|
|
Enabled: true,
|
|
OlderThan: &olderThan,
|
|
}, PrunePlanOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("PlanPrune() error = %v", err)
|
|
}
|
|
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("pruned = %q, want %q", got, want)
|
|
}
|
|
if got, want := pruneRecordPaths(report.PreservedOutputs), "fresh.txt"; got != want {
|
|
t.Fatalf("preserved = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
keepLatest := 0
|
|
sharedRoot := pruneSharedRootState(now)
|
|
|
|
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
|
|
Enabled: true,
|
|
KeepLatest: &keepLatest,
|
|
}, PrunePlanOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("PlanPrune() error = %v", err)
|
|
}
|
|
if got, want := report.CheckedCount, 1; got != want {
|
|
t.Fatalf("checked count = %d, want %d", got, want)
|
|
}
|
|
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
|
|
t.Fatalf("pruned = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
backend := fake.New()
|
|
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
|
original := pruneSingleOwnerState(now)
|
|
writeFakeSingleOwnerStateForPrune(t, backend, original)
|
|
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
|
|
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
DryRun: true,
|
|
Now: now,
|
|
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
|
if err != nil {
|
|
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
|
}
|
|
if !report.WouldChange || report.StateChanged || len(report.DeletedOutputs) != 0 {
|
|
t.Fatalf("report would_change=%t state_changed=%t deleted=%d, want dry-run only", report.WouldChange, report.StateChanged, len(report.DeletedOutputs))
|
|
}
|
|
if got, want := pruneRecordPaths(report.PlannedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("planned outputs = %q, want %q", got, want)
|
|
}
|
|
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
|
|
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
|
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
destinationState := readFakeSingleOwnerState(t, backend)
|
|
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
|
|
t.Fatalf("state outputs = %q, want original outputs", got)
|
|
}
|
|
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
|
|
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
|
|
}
|
|
}
|
|
|
|
func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
backend := fake.New()
|
|
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
|
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
|
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
|
|
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
|
if err != nil {
|
|
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
|
}
|
|
if !report.StateChanged || report.WouldChange {
|
|
t.Fatalf("report state_changed=%t would_change=%t, want applied change", report.StateChanged, report.WouldChange)
|
|
}
|
|
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
|
}
|
|
testutil.AssertFakeMissing(t, backend, "old.txt")
|
|
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
|
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
assertFakeStateExists(t, backend)
|
|
destinationState := readFakeSingleOwnerState(t, backend)
|
|
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
|
t.Fatalf("state outputs = %q, want fresh.txt", got)
|
|
}
|
|
if !destinationState.UpdatedAt.Equal(now) {
|
|
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
|
|
}
|
|
}
|
|
|
|
func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
backend := fake.New()
|
|
keepLatest := 0
|
|
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
|
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
|
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
|
|
|
|
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": failingBackend}))
|
|
if err == nil {
|
|
t.Fatal("pruneConfigWithBackendFactory() error = nil, want delete failure")
|
|
}
|
|
if report.FailedOutput == nil || report.FailedOutput.Path != "fresh.txt" {
|
|
t.Fatalf("failed output = %#v, want fresh.txt", report.FailedOutput)
|
|
}
|
|
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
|
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
|
}
|
|
testutil.AssertFakeMissing(t, backend, "old.txt")
|
|
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
|
assertFakeStateExists(t, backend)
|
|
destinationState := readFakeSingleOwnerState(t, backend)
|
|
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
|
t.Fatalf("state outputs = %q, want only failed output preserved", got)
|
|
}
|
|
}
|
|
|
|
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
|
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
|
backend := fake.New()
|
|
keepLatest := 0
|
|
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
|
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
|
|
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
|
|
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
Now: now,
|
|
}, fakeBackendFactoryProvider(t, map[string]storage.Backend{"s3:reports": backend}))
|
|
if err != nil {
|
|
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
|
}
|
|
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
|
|
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
|
}
|
|
testutil.AssertFakeMissing(t, backend, "archive.txt")
|
|
testutil.AssertFakeFile(t, backend, "html.txt", "old")
|
|
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
|
sharedRoot := readFakeSharedRootStateForApp(t, backend)
|
|
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
|
|
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
|
}
|
|
}
|
|
|
|
func pruneSingleOwnerState(now time.Time) state.DistributorState {
|
|
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
|
publishedAt := now.Add(-96 * time.Hour)
|
|
return state.DistributorState{
|
|
SchemaVersion: state.SchemaVersion,
|
|
PipelineID: "reports",
|
|
DestinationID: "archive",
|
|
PublishedAt: publishedAt,
|
|
CreatedAt: publishedAt,
|
|
UpdatedAt: publishedAt,
|
|
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
|
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
Source: state.SourceState{Manifest: manifest},
|
|
DistributorVersion: "test",
|
|
Outputs: []state.OutputFile{{
|
|
Path: "old.txt",
|
|
Kind: state.OutputKindSource,
|
|
SourcePath: "report.md",
|
|
SHA256: manifest.Files[0].SHA256,
|
|
Size: manifest.Files[0].Size,
|
|
CreatedAt: now.Add(-96 * time.Hour),
|
|
UpdatedAt: now.Add(-72 * time.Hour),
|
|
}, {
|
|
Path: "fresh.txt",
|
|
Kind: state.OutputKindSource,
|
|
SourcePath: "summary.txt",
|
|
SHA256: manifest.Files[1].SHA256,
|
|
Size: manifest.Files[1].Size,
|
|
CreatedAt: now.Add(-24 * time.Hour),
|
|
UpdatedAt: now.Add(-24 * time.Hour),
|
|
}},
|
|
}
|
|
}
|
|
|
|
func pruneSharedRootState(now time.Time) state.SharedRootState {
|
|
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
|
archive := state.CurrentOwnerScope("reports", "archive")
|
|
html := state.CurrentOwnerScope("reports", "html")
|
|
return state.SharedRootState{
|
|
SchemaVersion: state.SharedRootSchemaVersion,
|
|
DistributorVersion: "test",
|
|
CreatedAt: now.Add(-96 * time.Hour),
|
|
UpdatedAt: now.Add(-24 * time.Hour),
|
|
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
|
Owners: []state.OwnerRecord{{
|
|
Scope: archive,
|
|
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
Source: state.SourceState{Manifest: manifest},
|
|
}, {
|
|
Scope: html,
|
|
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
|
Source: state.SourceState{Manifest: manifest},
|
|
}},
|
|
Outputs: []state.SharedRootOutputFile{{
|
|
Path: "archive.txt",
|
|
Kind: state.OutputKindSource,
|
|
SourcePath: "report.md",
|
|
SHA256: manifest.Files[0].SHA256,
|
|
Size: manifest.Files[0].Size,
|
|
Owner: archive,
|
|
SourceID: manifest.ID,
|
|
SourceDigest: manifest.Digest,
|
|
SourceCreated: manifest.Created,
|
|
CreatedAt: now.Add(-96 * time.Hour),
|
|
UpdatedAt: now.Add(-72 * time.Hour),
|
|
}, {
|
|
Path: "html.txt",
|
|
Kind: state.OutputKindSource,
|
|
SourcePath: "summary.txt",
|
|
SHA256: manifest.Files[1].SHA256,
|
|
Size: manifest.Files[1].Size,
|
|
Owner: html,
|
|
SourceID: manifest.ID,
|
|
SourceDigest: manifest.Digest,
|
|
SourceCreated: manifest.Created,
|
|
CreatedAt: now.Add(-96 * time.Hour),
|
|
UpdatedAt: now.Add(-72 * time.Hour),
|
|
}},
|
|
}
|
|
}
|
|
|
|
func pruneRecordPaths(records []PruneOutputRecord) string {
|
|
paths := make([]string, 0, len(records))
|
|
for _, record := range records {
|
|
paths = append(paths, record.Path)
|
|
}
|
|
return strings.Join(paths, ",")
|
|
}
|
|
|
|
func pruneS3Config(t *testing.T, policy config.PrunePolicy) config.Config {
|
|
t.Helper()
|
|
cfg := config.Config{Pipelines: []config.Pipeline{{
|
|
ID: "reports",
|
|
Source: config.Backend{Backend: config.BackendLocal, Path: t.TempDir()},
|
|
Destinations: []config.Destination{{
|
|
ID: "archive",
|
|
Backend: config.BackendS3,
|
|
Bucket: "reports",
|
|
Retention: config.RetentionPolicy{
|
|
Prune: policy,
|
|
},
|
|
}},
|
|
}}}
|
|
config.ApplyDefaults(&cfg)
|
|
return cfg
|
|
}
|
|
|
|
func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
|
|
value := config.Duration(duration)
|
|
return config.PrunePolicy{
|
|
Enabled: true,
|
|
OlderThan: &value,
|
|
}
|
|
}
|
|
|
|
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
|
|
t.Helper()
|
|
data, err := json.MarshalIndent(destinationState, "", " ")
|
|
if err != nil {
|
|
t.Fatalf("marshal single-owner state: %v", err)
|
|
}
|
|
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
|
for _, output := range destinationState.Outputs {
|
|
testutil.WriteFakeFile(t, backend, output.Path, "managed")
|
|
}
|
|
}
|
|
|
|
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
|
|
t.Helper()
|
|
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {
|
|
t.Fatalf("state file stat error = %v", err)
|
|
}
|
|
}
|
|
|
|
type failingDeleteBackend struct {
|
|
storage.Backend
|
|
failPath string
|
|
}
|
|
|
|
func (b failingDeleteBackend) DeleteManagedOutputs(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
|
for _, path := range managedOutputPaths {
|
|
if path == b.failPath {
|
|
return storage.NewError(storage.OpDeleteManagedOutputs, "fake", path, storage.ErrPermission, nil)
|
|
}
|
|
}
|
|
return b.Backend.DeleteManagedOutputs(ctx, bundlePath, managedOutputPaths, opts)
|
|
}
|