Add prune retention planning

This commit is contained in:
2026-06-08 19:26:48 +00:00
parent 2abd09bde3
commit c67ecf86a9
12 changed files with 628 additions and 7 deletions

106
internal/app/prune.go Normal file
View File

@@ -0,0 +1,106 @@
package app
import (
"fmt"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
)
type PrunePlanOptions struct {
PipelineID string
DestinationID string
Now time.Time
}
type PrunePlanReport struct {
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
OwnerScope PruneOwnerScope `json:"owner_scope"`
Enabled bool `json:"enabled"`
CheckedCount int `json:"checked_count"`
PrunedOutputs []PruneOutputRecord `json:"pruned_outputs"`
PreservedOutputs []PruneOutputRecord `json:"preserved_outputs"`
}
type PruneOwnerScope struct {
PipelineID string `json:"pipeline_id"`
DestinationID string `json:"destination_id"`
}
type PruneOutputRecord struct {
Path string `json:"path"`
UpdatedAt string `json:"updated_at"`
Owner *PruneOwnerScope `json:"owner,omitempty"`
}
func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options PrunePlanOptions) (PrunePlanReport, error) {
scope := state.CurrentOwnerScope(options.PipelineID, options.DestinationID)
report := PrunePlanReport{
PipelineID: options.PipelineID,
DestinationID: options.DestinationID,
OwnerScope: PruneOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID},
Enabled: policy.Enabled,
PrunedOutputs: []PruneOutputRecord{},
PreservedOutputs: []PruneOutputRecord{},
}
if !policy.Enabled {
return report, nil
}
candidates, err := pruneCandidatesForDocument(document, scope)
if err != nil {
return PrunePlanReport{}, err
}
report.CheckedCount = len(candidates)
plan := state.PlanPrune(candidates, state.PrunePlanOptions{
Now: options.Now,
OlderThan: pruneOlderThan(policy),
KeepLatest: policy.KeepLatest,
})
report.PrunedOutputs = pruneOutputRecords(plan.Pruned)
report.PreservedOutputs = pruneOutputRecords(plan.Preserved)
return report, nil
}
func pruneCandidatesForDocument(document state.StateDocument, scope state.OwnerScope) ([]state.PruneCandidate, error) {
if document.SingleOwner != nil {
singleOwner := *document.SingleOwner
if singleOwner.PipelineID != scope.PipelineID || singleOwner.DestinationID != scope.DestinationID {
return nil, fmt.Errorf("state owner is %s/%s, not %s/%s", singleOwner.PipelineID, singleOwner.DestinationID, scope.PipelineID, scope.DestinationID)
}
return state.SingleOwnerPruneCandidates(singleOwner), nil
}
if document.SharedRoot != nil {
return state.SharedRootPruneCandidates(*document.SharedRoot, scope), nil
}
return nil, fmt.Errorf("destination state document is empty")
}
func pruneOlderThan(policy config.PrunePolicy) *time.Duration {
if policy.OlderThan == nil {
return nil
}
duration := policy.OlderThan.AsDuration()
return &duration
}
func pruneOutputRecords(candidates []state.PruneCandidate) []PruneOutputRecord {
records := make([]PruneOutputRecord, 0, len(candidates))
for _, candidate := range candidates {
var owner *PruneOwnerScope
if candidate.Owner != nil {
owner = &PruneOwnerScope{
PipelineID: candidate.Owner.PipelineID,
DestinationID: candidate.Owner.DestinationID,
}
}
records = append(records, PruneOutputRecord{
Path: candidate.Path,
UpdatedAt: candidate.UpdatedAt.UTC().Format(time.RFC3339),
Owner: owner,
})
}
return records
}

162
internal/app/prune_test.go Normal file
View File

@@ -0,0 +1,162 @@
package app
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
)
func TestPlanPruneDisabledPolicy(t *testing.T) {
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
PipelineID: "reports",
DestinationID: "archive",
})
if err != nil {
t.Fatalf("PlanPrune() error = %v", err)
}
if report.Enabled || report.CheckedCount != 0 || len(report.PrunedOutputs) != 0 {
t.Fatalf("report = %#v, want disabled empty plan", report)
}
}
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
olderThan := config.Duration(48 * time.Hour)
destinationState := pruneSingleOwnerState(now)
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
Enabled: true,
OlderThan: &olderThan,
}, PrunePlanOptions{
PipelineID: "reports",
DestinationID: "archive",
Now: now,
})
if err != nil {
t.Fatalf("PlanPrune() error = %v", err)
}
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
if got, want := pruneRecordPaths(report.PreservedOutputs), "fresh.txt"; got != want {
t.Fatalf("preserved = %q, want %q", got, want)
}
}
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
keepLatest := 0
sharedRoot := pruneSharedRootState(now)
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
Enabled: true,
KeepLatest: &keepLatest,
}, PrunePlanOptions{
PipelineID: "reports",
DestinationID: "archive",
Now: now,
})
if err != nil {
t.Fatalf("PlanPrune() error = %v", err)
}
if got, want := report.CheckedCount, 1; got != want {
t.Fatalf("checked count = %d, want %d", got, want)
}
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
}
func pruneSingleOwnerState(now time.Time) state.DistributorState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
publishedAt := now.Add(-96 * time.Hour)
return state.DistributorState{
SchemaVersion: state.SchemaVersion,
PipelineID: "reports",
DestinationID: "archive",
PublishedAt: publishedAt,
CreatedAt: publishedAt,
UpdatedAt: publishedAt,
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
DistributorVersion: "test",
Outputs: []state.OutputFile{{
Path: "old.txt",
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}, {
Path: "fresh.txt",
Kind: state.OutputKindSource,
SourcePath: "summary.txt",
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
CreatedAt: now.Add(-24 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
}},
}
}
func pruneSharedRootState(now time.Time) state.SharedRootState {
manifest := testutil.ValidManifest(testutil.BundleOptions{})
archive := state.CurrentOwnerScope("reports", "archive")
html := state.CurrentOwnerScope("reports", "html")
return state.SharedRootState{
SchemaVersion: state.SharedRootSchemaVersion,
DistributorVersion: "test",
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-24 * time.Hour),
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
Owners: []state.OwnerRecord{{
Scope: archive,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
}, {
Scope: html,
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
Source: state.SourceState{Manifest: manifest},
}},
Outputs: []state.SharedRootOutputFile{{
Path: "archive.txt",
Kind: state.OutputKindSource,
SourcePath: "report.md",
SHA256: manifest.Files[0].SHA256,
Size: manifest.Files[0].Size,
Owner: archive,
SourceID: manifest.ID,
SourceDigest: manifest.Digest,
SourceCreated: manifest.Created,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}, {
Path: "html.txt",
Kind: state.OutputKindSource,
SourcePath: "summary.txt",
SHA256: manifest.Files[1].SHA256,
Size: manifest.Files[1].Size,
Owner: html,
SourceID: manifest.ID,
SourceDigest: manifest.Digest,
SourceCreated: manifest.Created,
CreatedAt: now.Add(-96 * time.Hour),
UpdatedAt: now.Add(-72 * time.Hour),
}},
}
}
func pruneRecordPaths(records []PruneOutputRecord) string {
paths := make([]string, 0, len(records))
for _, record := range records {
paths = append(paths, record.Path)
}
return strings.Join(paths, ",")
}

View File

@@ -57,6 +57,7 @@ type Destination struct {
Links *Links `yaml:"links"`
State StatePolicy `yaml:"state"`
Reconciliation ReconciliationPolicy `yaml:"reconciliation"`
Retention RetentionPolicy `yaml:"retention"`
Transfer TransferPolicy `yaml:"transfer"`
}
@@ -128,6 +129,16 @@ type StatePolicy struct {
Mode string `yaml:"mode"`
}
type RetentionPolicy struct {
Prune PrunePolicy `yaml:"prune"`
}
type PrunePolicy struct {
Enabled bool `yaml:"enabled"`
OlderThan *Duration `yaml:"older_than"`
KeepLatest *int `yaml:"keep_latest"`
}
type TransferPolicy struct {
OnDestinationSame string `yaml:"on_destination_same"`
OnDestinationOlder string `yaml:"on_destination_older"`

View File

@@ -39,6 +39,9 @@ pipelines:
if got, want := destination.State.Mode, StateModeSingleOwner; got != want {
t.Fatalf("state mode default = %q, want %q", got, want)
}
if destination.Retention.Prune.Enabled {
t.Fatal("retention.prune.enabled default = true, want false")
}
if cfg.Secrets.Directory != "" {
t.Fatalf("secrets.directory = %q, want empty", cfg.Secrets.Directory)
}
@@ -228,6 +231,36 @@ pipelines:
}
}
func TestLoadFileAcceptsRetentionPruneConfig(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
- id: reports
source:
backend: local
path: /source
destinations:
- id: archive
backend: local
path: /archive
retention:
prune:
enabled: true
older_than: 168h
keep_latest: 3
`)
prune := cfg.Pipelines[0].Destinations[0].Retention.Prune
if !prune.Enabled {
t.Fatal("retention.prune.enabled = false, want true")
}
if prune.OlderThan == nil || prune.OlderThan.String() != "168h0m0s" {
t.Fatalf("retention.prune.older_than = %v, want 168h", prune.OlderThan)
}
if prune.KeepLatest == nil || *prune.KeepLatest != 3 {
t.Fatalf("retention.prune.keep_latest = %v, want 3", prune.KeepLatest)
}
}
func TestLoadFileAcceptsFixedPathMapping(t *testing.T) {
cfg := loadConfig(t, `
pipelines:
@@ -705,6 +738,7 @@ pipelines: [{id: reports, source: {backend: http_upload, max_upload_size: 20XB},
pipelines: [{id: reports, source: {backend: http_upload, max_upload_size: 0B}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"server duration": `server: {http: {retention: forever}}`,
"zero server duration": `server: {http: {retention: 0s}}`,
"prune duration": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, retention: {prune: {enabled: true, older_than: forever}}}]}]`,
"missing upload tokens": `pipelines: [{id: reports, source: {backend: http_upload}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
"destination http upload": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: ingest, backend: http_upload}]}]`,
"literal token": `upload_tokens: [{id: reporter, token: secret, token_env: UPLOAD_TOKEN, allow_pipelines: [reports]}]

View File

@@ -74,6 +74,7 @@ func Validate(cfg Config) error {
errs = validateLinks(errs, destinationContext+".links", destination.Links)
errs = validateStatePolicy(errs, destinationContext+".state", destination.State)
errs = validateReconciliationPolicy(errs, destinationContext+".reconciliation", destination.Reconciliation)
errs = validateRetentionPolicy(errs, destinationContext+".retention", destination.Retention)
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
}
}
@@ -323,6 +324,23 @@ func validateReconciliationPolicy(errs ValidationErrors, context string, policy
return errs
}
func validateRetentionPolicy(errs ValidationErrors, context string, policy RetentionPolicy) ValidationErrors {
prune := policy.Prune
if !prune.Enabled {
return errs
}
if prune.OlderThan == nil && prune.KeepLatest == nil {
errs = append(errs, context+".prune must set older_than or keep_latest when enabled is true")
}
if prune.OlderThan != nil && *prune.OlderThan <= 0 {
errs = append(errs, context+".prune.older_than must be greater than zero")
}
if prune.KeepLatest != nil && *prune.KeepLatest < 0 {
errs = append(errs, context+".prune.keep_latest must be zero or greater")
}
return errs
}
func validateStatePolicy(errs ValidationErrors, context string, policy StatePolicy) ValidationErrors {
if policy.Mode != StateModeSingleOwner && policy.Mode != StateModeSharedRoot {
errs = append(errs, context+".mode must be "+StateModeSingleOwner+" or "+StateModeSharedRoot)

View File

@@ -3,6 +3,7 @@ package config
import (
"strings"
"testing"
"time"
)
func TestValidatePublishTransformPolicy(t *testing.T) {
@@ -175,6 +176,50 @@ func TestValidateStatePolicy(t *testing.T) {
}
}
func TestValidateRetentionPolicy(t *testing.T) {
olderThan := Duration(24 * time.Hour)
zeroDuration := Duration(0)
keepZero := 0
keepThree := 3
keepNegative := -1
tests := []struct {
name string
retention RetentionPolicy
wantErr bool
}{
{name: "disabled"},
{name: "older than", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, OlderThan: &olderThan}}},
{name: "keep zero", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, KeepLatest: &keepZero}}},
{name: "keep latest", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, KeepLatest: &keepThree}}},
{name: "combined", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, OlderThan: &olderThan, KeepLatest: &keepThree}}},
{name: "missing policy", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true}}, wantErr: true},
{name: "zero older than", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, OlderThan: &zeroDuration}}, wantErr: true},
{name: "negative keep latest", retention: RetentionPolicy{Prune: PrunePolicy{Enabled: true, KeepLatest: &keepNegative}}, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Config{Pipelines: []Pipeline{{
ID: "reports",
Source: Backend{Backend: BackendLocal, Path: "/source"},
Destinations: []Destination{{
ID: "archive",
Backend: BackendLocal,
Path: "/destination",
Retention: tt.retention,
}},
}}}
ApplyDefaults(&cfg)
err := Validate(cfg)
if tt.wantErr && err == nil {
t.Fatal("Validate() error = nil, want error")
}
if !tt.wantErr && err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestValidateLinks(t *testing.T) {
tests := []struct {
name string

121
internal/state/prune.go Normal file
View File

@@ -0,0 +1,121 @@
package state
import (
"sort"
"time"
)
type PruneCandidate struct {
Path string
UpdatedAt time.Time
Owner *OwnerScope
}
type PrunePlanOptions struct {
Now time.Time
OlderThan *time.Duration
KeepLatest *int
}
type PrunePlan struct {
Pruned []PruneCandidate
Preserved []PruneCandidate
}
func SingleOwnerPruneCandidates(s DistributorState) []PruneCandidate {
candidates := make([]PruneCandidate, 0, len(s.Outputs))
for _, output := range s.Outputs {
candidates = append(candidates, PruneCandidate{
Path: output.Path,
UpdatedAt: output.UpdatedAt,
})
}
return candidates
}
func SharedRootPruneCandidates(s SharedRootState, scope OwnerScope) []PruneCandidate {
candidates := make([]PruneCandidate, 0, len(s.Outputs))
for _, output := range s.Outputs {
if output.Owner != scope {
continue
}
owner := output.Owner
candidates = append(candidates, PruneCandidate{
Path: output.Path,
UpdatedAt: output.UpdatedAt,
Owner: &owner,
})
}
return candidates
}
func PlanPrune(candidates []PruneCandidate, options PrunePlanOptions) PrunePlan {
ordered := append([]PruneCandidate(nil), candidates...)
sortPruneCandidatesNewestFirst(ordered)
if options.OlderThan == nil && options.KeepLatest == nil {
return PrunePlan{
Pruned: []PruneCandidate{},
Preserved: ordered,
}
}
preservedByPath := make(map[string]struct{})
if options.KeepLatest != nil {
keep := *options.KeepLatest
if keep < 0 {
keep = 0
}
if keep > len(ordered) {
keep = len(ordered)
}
for _, candidate := range ordered[:keep] {
preservedByPath[candidate.Path] = struct{}{}
}
}
plan := PrunePlan{
Pruned: []PruneCandidate{},
Preserved: []PruneCandidate{},
}
cutoff := time.Time{}
if options.OlderThan != nil {
now := options.Now.UTC()
if now.IsZero() {
now = time.Now().UTC()
}
cutoff = now.Add(-*options.OlderThan)
}
for _, candidate := range ordered {
if _, preserved := preservedByPath[candidate.Path]; preserved {
plan.Preserved = append(plan.Preserved, candidate)
continue
}
if options.OlderThan == nil || candidate.UpdatedAt.Before(cutoff) {
plan.Pruned = append(plan.Pruned, candidate)
continue
}
plan.Preserved = append(plan.Preserved, candidate)
}
sortPruneCandidatesOldestFirst(plan.Pruned)
sortPruneCandidatesNewestFirst(plan.Preserved)
return plan
}
func sortPruneCandidatesNewestFirst(candidates []PruneCandidate) {
sort.Slice(candidates, func(i, j int) bool {
if !candidates[i].UpdatedAt.Equal(candidates[j].UpdatedAt) {
return candidates[i].UpdatedAt.After(candidates[j].UpdatedAt)
}
return candidates[i].Path < candidates[j].Path
})
}
func sortPruneCandidatesOldestFirst(candidates []PruneCandidate) {
sort.Slice(candidates, func(i, j int) bool {
if !candidates[i].UpdatedAt.Equal(candidates[j].UpdatedAt) {
return candidates[i].UpdatedAt.Before(candidates[j].UpdatedAt)
}
return candidates[i].Path < candidates[j].Path
})
}

View File

@@ -0,0 +1,96 @@
package state
import (
"strings"
"testing"
"time"
)
func TestPlanPruneOlderThan(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
olderThan := 48 * time.Hour
plan := PlanPrune([]PruneCandidate{
{Path: "old.txt", UpdatedAt: now.Add(-72 * time.Hour)},
{Path: "fresh.txt", UpdatedAt: now.Add(-24 * time.Hour)},
}, PrunePlanOptions{Now: now, OlderThan: &olderThan})
if got, want := pruneCandidatePaths(plan.Pruned), "old.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
if got, want := pruneCandidatePaths(plan.Preserved), "fresh.txt"; got != want {
t.Fatalf("preserved = %q, want %q", got, want)
}
}
func TestPlanPruneKeepLatest(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
keepLatest := 2
plan := PlanPrune([]PruneCandidate{
{Path: "old.txt", UpdatedAt: now.Add(-72 * time.Hour)},
{Path: "new.txt", UpdatedAt: now.Add(-1 * time.Hour)},
{Path: "middle.txt", UpdatedAt: now.Add(-24 * time.Hour)},
}, PrunePlanOptions{KeepLatest: &keepLatest})
if got, want := pruneCandidatePaths(plan.Pruned), "old.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
if got, want := pruneCandidatePaths(plan.Preserved), "new.txt,middle.txt"; got != want {
t.Fatalf("preserved = %q, want %q", got, want)
}
}
func TestPlanPruneCombinedPolicyPreservesLatestBeforeAgeCheck(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
olderThan := 48 * time.Hour
keepLatest := 1
plan := PlanPrune([]PruneCandidate{
{Path: "oldest.txt", UpdatedAt: now.Add(-96 * time.Hour)},
{Path: "old.txt", UpdatedAt: now.Add(-72 * time.Hour)},
{Path: "fresh.txt", UpdatedAt: now.Add(-24 * time.Hour)},
}, PrunePlanOptions{Now: now, OlderThan: &olderThan, KeepLatest: &keepLatest})
if got, want := pruneCandidatePaths(plan.Pruned), "oldest.txt,old.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
if got, want := pruneCandidatePaths(plan.Preserved), "fresh.txt"; got != want {
t.Fatalf("preserved = %q, want %q", got, want)
}
}
func TestPlanPruneDeterministicTieBreaking(t *testing.T) {
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
keepLatest := 1
plan := PlanPrune([]PruneCandidate{
{Path: "b.txt", UpdatedAt: now},
{Path: "a.txt", UpdatedAt: now},
{Path: "c.txt", UpdatedAt: now.Add(-time.Hour)},
}, PrunePlanOptions{KeepLatest: &keepLatest})
if got, want := pruneCandidatePaths(plan.Preserved), "a.txt"; got != want {
t.Fatalf("preserved = %q, want %q", got, want)
}
if got, want := pruneCandidatePaths(plan.Pruned), "c.txt,b.txt"; got != want {
t.Fatalf("pruned = %q, want %q", got, want)
}
}
func TestSharedRootPruneCandidatesPreserveOtherOwners(t *testing.T) {
sharedRoot := validSharedRootState(t)
scope := CurrentOwnerScope("reports", "archive")
candidates := SharedRootPruneCandidates(sharedRoot, scope)
if got, want := pruneCandidatePaths(candidates), "report.md"; got != want {
t.Fatalf("candidates = %q, want %q", got, want)
}
if candidates[0].Owner == nil || *candidates[0].Owner != scope {
t.Fatalf("candidate owner = %#v, want current owner", candidates[0].Owner)
}
}
func pruneCandidatePaths(candidates []PruneCandidate) string {
paths := make([]string, 0, len(candidates))
for _, candidate := range candidates {
paths = append(paths, candidate.Path)
}
return strings.Join(paths, ",")
}