Add prune retention planning

2026-06-08 19:26:48 +00:00
parent 2abd09bde3
commit c67ecf86a9
12 changed files with 628 additions and 7 deletions
--- a/internal/app/prune.go
+++ b/internal/app/prune.go
@@ -0,0 +1,106 @@
+package app
+
+import (
+	"fmt"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/state"
+)
+
+type PrunePlanOptions struct {
+	PipelineID    string
+	DestinationID string
+	Now           time.Time
+}
+
+type PrunePlanReport struct {
+	PipelineID       string              `json:"pipeline_id"`
+	DestinationID    string              `json:"destination_id"`
+	OwnerScope       PruneOwnerScope     `json:"owner_scope"`
+	Enabled          bool                `json:"enabled"`
+	CheckedCount     int                 `json:"checked_count"`
+	PrunedOutputs    []PruneOutputRecord `json:"pruned_outputs"`
+	PreservedOutputs []PruneOutputRecord `json:"preserved_outputs"`
+}
+
+type PruneOwnerScope struct {
+	PipelineID    string `json:"pipeline_id"`
+	DestinationID string `json:"destination_id"`
+}
+
+type PruneOutputRecord struct {
+	Path      string           `json:"path"`
+	UpdatedAt string           `json:"updated_at"`
+	Owner     *PruneOwnerScope `json:"owner,omitempty"`
+}
+
+func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options PrunePlanOptions) (PrunePlanReport, error) {
+	scope := state.CurrentOwnerScope(options.PipelineID, options.DestinationID)
+	report := PrunePlanReport{
+		PipelineID:       options.PipelineID,
+		DestinationID:    options.DestinationID,
+		OwnerScope:       PruneOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID},
+		Enabled:          policy.Enabled,
+		PrunedOutputs:    []PruneOutputRecord{},
+		PreservedOutputs: []PruneOutputRecord{},
+	}
+	if !policy.Enabled {
+		return report, nil
+	}
+
+	candidates, err := pruneCandidatesForDocument(document, scope)
+	if err != nil {
+		return PrunePlanReport{}, err
+	}
+	report.CheckedCount = len(candidates)
+	plan := state.PlanPrune(candidates, state.PrunePlanOptions{
+		Now:        options.Now,
+		OlderThan:  pruneOlderThan(policy),
+		KeepLatest: policy.KeepLatest,
+	})
+	report.PrunedOutputs = pruneOutputRecords(plan.Pruned)
+	report.PreservedOutputs = pruneOutputRecords(plan.Preserved)
+	return report, nil
+}
+
+func pruneCandidatesForDocument(document state.StateDocument, scope state.OwnerScope) ([]state.PruneCandidate, error) {
+	if document.SingleOwner != nil {
+		singleOwner := *document.SingleOwner
+		if singleOwner.PipelineID != scope.PipelineID || singleOwner.DestinationID != scope.DestinationID {
+			return nil, fmt.Errorf("state owner is %s/%s, not %s/%s", singleOwner.PipelineID, singleOwner.DestinationID, scope.PipelineID, scope.DestinationID)
+		}
+		return state.SingleOwnerPruneCandidates(singleOwner), nil
+	}
+	if document.SharedRoot != nil {
+		return state.SharedRootPruneCandidates(*document.SharedRoot, scope), nil
+	}
+	return nil, fmt.Errorf("destination state document is empty")
+}
+
+func pruneOlderThan(policy config.PrunePolicy) *time.Duration {
+	if policy.OlderThan == nil {
+		return nil
+	}
+	duration := policy.OlderThan.AsDuration()
+	return &duration
+}
+
+func pruneOutputRecords(candidates []state.PruneCandidate) []PruneOutputRecord {
+	records := make([]PruneOutputRecord, 0, len(candidates))
+	for _, candidate := range candidates {
+		var owner *PruneOwnerScope
+		if candidate.Owner != nil {
+			owner = &PruneOwnerScope{
+				PipelineID:    candidate.Owner.PipelineID,
+				DestinationID: candidate.Owner.DestinationID,
+			}
+		}
+		records = append(records, PruneOutputRecord{
+			Path:      candidate.Path,
+			UpdatedAt: candidate.UpdatedAt.UTC().Format(time.RFC3339),
+			Owner:     owner,
+		})
+	}
+	return records
+}
--- a/internal/app/prune_test.go
+++ b/internal/app/prune_test.go
@@ -0,0 +1,162 @@
+package app
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/distributor/internal/config"
+	"gitea.maximumdirect.net/eric/distributor/internal/state"
+	"gitea.maximumdirect.net/eric/distributor/internal/testutil"
+)
+
+func TestPlanPruneDisabledPolicy(t *testing.T) {
+	document := state.StateDocument{SingleOwner: &state.DistributorState{}}
+	report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
+		PipelineID:    "reports",
+		DestinationID: "archive",
+	})
+	if err != nil {
+		t.Fatalf("PlanPrune() error = %v", err)
+	}
+	if report.Enabled || report.CheckedCount != 0 || len(report.PrunedOutputs) != 0 {
+		t.Fatalf("report = %#v, want disabled empty plan", report)
+	}
+}
+
+func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
+	now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
+	olderThan := config.Duration(48 * time.Hour)
+	destinationState := pruneSingleOwnerState(now)
+
+	report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
+		Enabled:   true,
+		OlderThan: &olderThan,
+	}, PrunePlanOptions{
+		PipelineID:    "reports",
+		DestinationID: "archive",
+		Now:           now,
+	})
+	if err != nil {
+		t.Fatalf("PlanPrune() error = %v", err)
+	}
+	if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt"; got != want {
+		t.Fatalf("pruned = %q, want %q", got, want)
+	}
+	if got, want := pruneRecordPaths(report.PreservedOutputs), "fresh.txt"; got != want {
+		t.Fatalf("preserved = %q, want %q", got, want)
+	}
+}
+
+func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
+	now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
+	keepLatest := 0
+	sharedRoot := pruneSharedRootState(now)
+
+	report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
+		Enabled:    true,
+		KeepLatest: &keepLatest,
+	}, PrunePlanOptions{
+		PipelineID:    "reports",
+		DestinationID: "archive",
+		Now:           now,
+	})
+	if err != nil {
+		t.Fatalf("PlanPrune() error = %v", err)
+	}
+	if got, want := report.CheckedCount, 1; got != want {
+		t.Fatalf("checked count = %d, want %d", got, want)
+	}
+	if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
+		t.Fatalf("pruned = %q, want %q", got, want)
+	}
+}
+
+func pruneSingleOwnerState(now time.Time) state.DistributorState {
+	manifest := testutil.ValidManifest(testutil.BundleOptions{})
+	publishedAt := now.Add(-96 * time.Hour)
+	return state.DistributorState{
+		SchemaVersion:      state.SchemaVersion,
+		PipelineID:         "reports",
+		DestinationID:      "archive",
+		PublishedAt:        publishedAt,
+		CreatedAt:          publishedAt,
+		UpdatedAt:          publishedAt,
+		State:              state.StatePolicy{Mode: state.StateModeSingleOwner},
+		Reconciliation:     state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
+		Source:             state.SourceState{Manifest: manifest},
+		DistributorVersion: "test",
+		Outputs: []state.OutputFile{{
+			Path:       "old.txt",
+			Kind:       state.OutputKindSource,
+			SourcePath: "report.md",
+			SHA256:     manifest.Files[0].SHA256,
+			Size:       manifest.Files[0].Size,
+			CreatedAt:  now.Add(-96 * time.Hour),
+			UpdatedAt:  now.Add(-72 * time.Hour),
+		}, {
+			Path:       "fresh.txt",
+			Kind:       state.OutputKindSource,
+			SourcePath: "summary.txt",
+			SHA256:     manifest.Files[1].SHA256,
+			Size:       manifest.Files[1].Size,
+			CreatedAt:  now.Add(-24 * time.Hour),
+			UpdatedAt:  now.Add(-24 * time.Hour),
+		}},
+	}
+}
+
+func pruneSharedRootState(now time.Time) state.SharedRootState {
+	manifest := testutil.ValidManifest(testutil.BundleOptions{})
+	archive := state.CurrentOwnerScope("reports", "archive")
+	html := state.CurrentOwnerScope("reports", "html")
+	return state.SharedRootState{
+		SchemaVersion:      state.SharedRootSchemaVersion,
+		DistributorVersion: "test",
+		CreatedAt:          now.Add(-96 * time.Hour),
+		UpdatedAt:          now.Add(-24 * time.Hour),
+		State:              state.StatePolicy{Mode: state.StateModeSharedRoot},
+		Owners: []state.OwnerRecord{{
+			Scope:          archive,
+			Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
+			Source:         state.SourceState{Manifest: manifest},
+		}, {
+			Scope:          html,
+			Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
+			Source:         state.SourceState{Manifest: manifest},
+		}},
+		Outputs: []state.SharedRootOutputFile{{
+			Path:          "archive.txt",
+			Kind:          state.OutputKindSource,
+			SourcePath:    "report.md",
+			SHA256:        manifest.Files[0].SHA256,
+			Size:          manifest.Files[0].Size,
+			Owner:         archive,
+			SourceID:      manifest.ID,
+			SourceDigest:  manifest.Digest,
+			SourceCreated: manifest.Created,
+			CreatedAt:     now.Add(-96 * time.Hour),
+			UpdatedAt:     now.Add(-72 * time.Hour),
+		}, {
+			Path:          "html.txt",
+			Kind:          state.OutputKindSource,
+			SourcePath:    "summary.txt",
+			SHA256:        manifest.Files[1].SHA256,
+			Size:          manifest.Files[1].Size,
+			Owner:         html,
+			SourceID:      manifest.ID,
+			SourceDigest:  manifest.Digest,
+			SourceCreated: manifest.Created,
+			CreatedAt:     now.Add(-96 * time.Hour),
+			UpdatedAt:     now.Add(-72 * time.Hour),
+		}},
+	}
+}
+
+func pruneRecordPaths(records []PruneOutputRecord) string {
+	paths := make([]string, 0, len(records))
+	for _, record := range records {
+		paths = append(paths, record.Path)
+	}
+	return strings.Join(paths, ",")
+}