Support catalog maintenance commands
This commit is contained in:
@@ -211,24 +211,13 @@ func removePrunedStateRecords(ctx context.Context, backend storage.Backend, stat
|
||||
if len(paths) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
if document.SingleOwner != nil {
|
||||
next, changed := state.RemoveMissingOutputs(*document.SingleOwner, paths)
|
||||
if document.Catalog != nil {
|
||||
next, changed := state.RemoveMissingCatalogOwnerOutputs(*document.Catalog, scope, paths)
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
next.UpdatedAt = now
|
||||
if err := state.Validate(next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, writeRepairedState(ctx, backend, statePath, next)
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
next, changed := state.RemoveMissingSharedRootOwnerOutputs(*document.SharedRoot, scope, paths)
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
next.UpdatedAt = now
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
if err := state.ValidateCatalog(next); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, writeRepairedState(ctx, backend, statePath, next)
|
||||
@@ -266,15 +255,8 @@ func PlanPrune(document state.StateDocument, policy config.PrunePolicy, options
|
||||
}
|
||||
|
||||
func pruneCandidatesForDocument(document state.StateDocument, scope state.OwnerScope) ([]state.PruneCandidate, error) {
|
||||
if document.SingleOwner != nil {
|
||||
singleOwner := *document.SingleOwner
|
||||
if singleOwner.PipelineID != scope.PipelineID || singleOwner.DestinationID != scope.DestinationID {
|
||||
return nil, fmt.Errorf("state owner is %s/%s, not %s/%s", singleOwner.PipelineID, singleOwner.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
return state.SingleOwnerPruneCandidates(singleOwner), nil
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
return state.SharedRootPruneCandidates(*document.SharedRoot, scope), nil
|
||||
if document.Catalog != nil {
|
||||
return state.CatalogPruneCandidates(*document.Catalog, scope), nil
|
||||
}
|
||||
return nil, unsupportedStateDocumentError(document)
|
||||
}
|
||||
@@ -283,8 +265,8 @@ func unsupportedStateDocumentError(document state.StateDocument) error {
|
||||
if document.SupersededLegacy != nil {
|
||||
return fmt.Errorf("destination state schema_version %d is superseded legacy state", document.SupersededLegacy.SchemaVersion)
|
||||
}
|
||||
if document.Catalog != nil {
|
||||
return fmt.Errorf("catalog destination state is not supported by this command")
|
||||
if document.SingleOwner != nil || document.SharedRoot != nil {
|
||||
return fmt.Errorf("legacy destination state is not supported by this command")
|
||||
}
|
||||
return fmt.Errorf("destination state document is empty")
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
)
|
||||
|
||||
func TestPlanPruneDisabledPolicy(t *testing.T) {
|
||||
document := state.StateDocument{SingleOwner: &state.DistributorState{}}
|
||||
document := state.StateDocument{Catalog: &state.CatalogState{}}
|
||||
report, err := PlanPrune(document, config.PrunePolicy{}, PrunePlanOptions{
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
@@ -28,12 +28,12 @@ func TestPlanPruneDisabledPolicy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
|
||||
func TestPlanPruneCatalogOutputs(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
olderThan := config.Duration(48 * time.Hour)
|
||||
destinationState := pruneSingleOwnerState(now)
|
||||
catalog := pruneCatalogState(now)
|
||||
|
||||
report, err := PlanPrune(state.StateDocument{SingleOwner: &destinationState}, config.PrunePolicy{
|
||||
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
|
||||
Enabled: true,
|
||||
OlderThan: &olderThan,
|
||||
}, PrunePlanOptions{
|
||||
@@ -52,12 +52,12 @@ func TestPlanPruneSingleOwnerOutputs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
|
||||
func TestPlanPruneCatalogCurrentOwnerOnly(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
keepLatest := 0
|
||||
sharedRoot := pruneSharedRootState(now)
|
||||
catalog := pruneCatalogState(now)
|
||||
|
||||
report, err := PlanPrune(state.StateDocument{SharedRoot: &sharedRoot}, config.PrunePolicy{
|
||||
report, err := PlanPrune(state.StateDocument{Catalog: &catalog}, config.PrunePolicy{
|
||||
Enabled: true,
|
||||
KeepLatest: &keepLatest,
|
||||
}, PrunePlanOptions{
|
||||
@@ -68,21 +68,20 @@ func TestPlanPruneSharedRootCurrentOwnerOnly(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("PlanPrune() error = %v", err)
|
||||
}
|
||||
if got, want := report.CheckedCount, 1; got != want {
|
||||
if got, want := report.CheckedCount, 2; got != want {
|
||||
t.Fatalf("checked count = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.PrunedOutputs), "archive.txt"; got != want {
|
||||
if got, want := pruneRecordPaths(report.PrunedOutputs), "old.txt,fresh.txt"; got != want {
|
||||
t.Fatalf("pruned = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *testing.T) {
|
||||
t.Skip("catalog prune execution is covered by the catalog maintenance work")
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
original := pruneSingleOwnerState(now)
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, original)
|
||||
original := pruneCatalogState(now)
|
||||
writeFakeCatalogState(t, backend, original)
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -103,21 +102,20 @@ func TestPruneDryRunReportsPlannedDeletesWithoutDeletingOrRewritingState(t *test
|
||||
testutil.AssertFakeFile(t, backend, "old.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "old.txt,fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "old.txt,fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
if !destinationState.UpdatedAt.Equal(original.UpdatedAt) {
|
||||
t.Fatalf("state updated_at = %s, want original %s", destinationState.UpdatedAt, original.UpdatedAt)
|
||||
if !catalog.UpdatedAt.Equal(original.UpdatedAt) {
|
||||
t.Fatalf("state updated_at = %s, want original %s", catalog.UpdatedAt, original.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
||||
t.Skip("catalog prune execution is covered by the catalog maintenance work")
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -136,24 +134,27 @@ func TestPruneApplyDeletesOnlyManagedOutputsAndUpdatesState(t *testing.T) {
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
assertFakeStateExists(t, backend)
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
if !destinationState.UpdatedAt.Equal(now) {
|
||||
t.Fatalf("state updated_at = %s, want %s", destinationState.UpdatedAt, now)
|
||||
if catalog.SchemaVersion != state.CatalogSchemaVersion {
|
||||
t.Fatalf("state schema_version = %d, want %d", catalog.SchemaVersion, state.CatalogSchemaVersion)
|
||||
}
|
||||
if !catalog.UpdatedAt.Equal(now) {
|
||||
t.Fatalf("state updated_at = %s, want %s", catalog.UpdatedAt, now)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
||||
t.Skip("catalog prune execution is covered by the catalog maintenance work")
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
keepLatest := 0
|
||||
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
||||
writeFakeSingleOwnerStateForPrune(t, backend, pruneSingleOwnerState(now))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
failingBackend := failingDeleteBackend{Backend: backend, failPath: "fresh.txt"}
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -173,19 +174,17 @@ func TestPruneApplyPreservesStateForFailedDeletes(t *testing.T) {
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "fresh.txt", "managed")
|
||||
assertFakeStateExists(t, backend)
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "fresh.txt" {
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want only failed output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
||||
t.Skip("catalog prune execution is covered by the catalog maintenance work")
|
||||
func TestPrunePreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.T) {
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
backend := fake.New()
|
||||
keepLatest := 0
|
||||
cfg := pruneS3Config(t, config.PrunePolicy{Enabled: true, KeepLatest: &keepLatest})
|
||||
writeFakeSharedRootStateForApp(t, backend, pruneSharedRootState(now))
|
||||
cfg := pruneS3Config(t, pruneOlderThanPolicy(48*time.Hour))
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(now))
|
||||
testutil.WriteFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
|
||||
report, err := pruneConfigWithBackendFactory(context.Background(), cfg, PruneOptions{
|
||||
@@ -196,93 +195,56 @@ func TestPruneSharedRootPreservesOtherOwnersWhenScopedToCurrentOwner(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("pruneConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "archive.txt"; got != want {
|
||||
if got, want := pruneRecordPaths(report.DeletedOutputs), "old.txt"; got != want {
|
||||
t.Fatalf("deleted outputs = %q, want %q", got, want)
|
||||
}
|
||||
testutil.AssertFakeMissing(t, backend, "archive.txt")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "old")
|
||||
testutil.AssertFakeMissing(t, backend, "old.txt")
|
||||
testutil.AssertFakeFile(t, backend, "html.txt", "managed")
|
||||
testutil.AssertFakeFile(t, backend, "unmanaged.txt", "keep")
|
||||
sharedRoot := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := strings.Join(sharedRoot.AllManagedOutputPaths(), ","); got != "html.txt" {
|
||||
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
||||
catalog := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(catalog), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func pruneSingleOwnerState(now time.Time) state.DistributorState {
|
||||
func pruneCatalogState(now time.Time) state.CatalogState {
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
publishedAt := now.Add(-96 * time.Hour)
|
||||
return state.DistributorState{
|
||||
SchemaVersion: state.SchemaVersion,
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
PublishedAt: publishedAt,
|
||||
CreatedAt: publishedAt,
|
||||
UpdatedAt: publishedAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSingleOwner},
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
createdAt := now.Add(-96 * time.Hour)
|
||||
source := state.CatalogSourceIdentity{ID: manifest.ID, Digest: manifest.Digest, Created: manifest.Created}
|
||||
return state.CatalogState{
|
||||
SchemaVersion: state.CatalogSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
Outputs: []state.OutputFile{{
|
||||
Path: "old.txt",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}, {
|
||||
Path: "fresh.txt",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
CreatedAt: now.Add(-24 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func pruneSharedRootState(now time.Time) state.SharedRootState {
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
archive := state.CurrentOwnerScope("reports", "archive")
|
||||
html := state.CurrentOwnerScope("reports", "html")
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: archive,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
}, {
|
||||
Scope: html,
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: manifest},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "archive.txt",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeCatalog},
|
||||
Outputs: []state.CatalogOutputFile{{
|
||||
Path: "old.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
Owner: archive,
|
||||
SourceID: manifest.ID,
|
||||
SourceDigest: manifest.Digest,
|
||||
SourceCreated: manifest.Created,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}, {
|
||||
Path: "html.txt",
|
||||
Path: "fresh.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "archive",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SHA256: manifest.Files[0].SHA256,
|
||||
Size: manifest.Files[0].Size,
|
||||
CreatedAt: now.Add(-24 * time.Hour),
|
||||
UpdatedAt: now.Add(-24 * time.Hour),
|
||||
}, {
|
||||
Path: "html.txt",
|
||||
PipelineID: "reports",
|
||||
DestinationID: "html",
|
||||
Source: source,
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "summary.txt",
|
||||
SHA256: manifest.Files[1].SHA256,
|
||||
Size: manifest.Files[1].Size,
|
||||
Owner: html,
|
||||
SourceID: manifest.ID,
|
||||
SourceDigest: manifest.Digest,
|
||||
SourceCreated: manifest.Created,
|
||||
CreatedAt: now.Add(-96 * time.Hour),
|
||||
UpdatedAt: now.Add(-72 * time.Hour),
|
||||
}},
|
||||
@@ -323,18 +285,31 @@ func pruneOlderThanPolicy(duration time.Duration) config.PrunePolicy {
|
||||
}
|
||||
}
|
||||
|
||||
func writeFakeSingleOwnerStateForPrune(t *testing.T, backend *fake.Backend, destinationState state.DistributorState) {
|
||||
func writeFakeCatalogState(t *testing.T, backend *fake.Backend, catalog state.CatalogState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||
data, err := json.MarshalIndent(catalog, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal single-owner state: %v", err)
|
||||
t.Fatalf("marshal catalog state: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
||||
for _, output := range destinationState.Outputs {
|
||||
for _, output := range catalog.Outputs {
|
||||
testutil.WriteFakeFile(t, backend, output.Path, "managed")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeCatalogState(t *testing.T, backend *fake.Backend) state.CatalogState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read catalog state: %v", err)
|
||||
}
|
||||
catalog, err := state.ParseCatalog(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse catalog state: %v", err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
func assertFakeStateExists(t *testing.T, backend *fake.Backend) {
|
||||
t.Helper()
|
||||
if _, err := backend.Stat(context.Background(), storage.StateFileName); err != nil {
|
||||
|
||||
@@ -158,65 +158,25 @@ func buildReconcileStateReport(ctx context.Context, backend storage.Backend, pip
|
||||
DryRun: options.DryRun,
|
||||
}
|
||||
scope := state.CurrentOwnerScope(pipeline.ID, destination.ID)
|
||||
if document.SingleOwner != nil {
|
||||
return reconcileSingleOwnerState(ctx, backend, statePath, *document.SingleOwner, scope, report, options)
|
||||
}
|
||||
if document.SharedRoot != nil {
|
||||
return reconcileSharedRootState(ctx, backend, statePath, *document.SharedRoot, scope, report, options)
|
||||
if document.Catalog != nil {
|
||||
return reconcileCatalogState(ctx, backend, statePath, *document.Catalog, scope, report, options)
|
||||
}
|
||||
return ReconcileStateReport{}, unsupportedStateDocumentError(document)
|
||||
}
|
||||
|
||||
func reconcileSingleOwnerState(ctx context.Context, backend storage.Backend, statePath string, destinationState state.DistributorState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
if destinationState.PipelineID != scope.PipelineID || destinationState.DestinationID != scope.DestinationID {
|
||||
return ReconcileStateReport{}, fmt.Errorf("state owner is %s/%s, not %s/%s", destinationState.PipelineID, destinationState.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
report.StateSchema = destinationState.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID}
|
||||
managed := state.ManagedOutputPaths(destinationState)
|
||||
missing, err := missingSingleOwnerOutputs(ctx, backend, destinationState.Outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.CheckedCount = len(managed)
|
||||
report.MissingManagedOutputs = missing
|
||||
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.UnmanagedEntries = unmanaged
|
||||
report.WouldChange = options.DryRun && len(missing) > 0
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
next, changed := state.RemoveMissingOutputs(destinationState, missingPaths)
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.Validate(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func reconcileSharedRootState(ctx context.Context, backend storage.Backend, statePath string, sharedRoot state.SharedRootState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
report.StateSchema = sharedRoot.SchemaVersion
|
||||
func reconcileCatalogState(ctx context.Context, backend storage.Backend, statePath string, catalog state.CatalogState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
report.StateSchema = catalog.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{
|
||||
PipelineID: scope.PipelineID,
|
||||
DestinationID: scope.DestinationID,
|
||||
AllOwners: options.AllOwners,
|
||||
}
|
||||
managed := sharedRoot.AllManagedOutputPaths()
|
||||
outputs := sharedRoot.Outputs
|
||||
managed := state.CatalogManagedOutputPaths(catalog)
|
||||
outputs := catalog.Outputs
|
||||
if !options.AllOwners {
|
||||
outputs = sharedRootOutputsForOwner(sharedRoot.Outputs, scope)
|
||||
outputs = state.CatalogOutputsForOwner(catalog.Outputs, scope)
|
||||
}
|
||||
missing, err := missingSharedRootOutputs(ctx, backend, outputs)
|
||||
missing, err := missingCatalogOutputs(ctx, backend, outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
@@ -231,17 +191,17 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
var next state.SharedRootState
|
||||
var next state.CatalogState
|
||||
var changed bool
|
||||
if options.AllOwners {
|
||||
next, changed = state.RemoveMissingSharedRootOutputs(sharedRoot, missingPaths)
|
||||
next, changed = state.RemoveMissingCatalogOutputs(catalog, missingPaths)
|
||||
} else {
|
||||
next, changed = state.RemoveMissingSharedRootOwnerOutputs(sharedRoot, scope, missingPaths)
|
||||
next, changed = state.RemoveMissingCatalogOwnerOutputs(catalog, scope, missingPaths)
|
||||
}
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
if err := state.ValidateCatalog(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
@@ -252,21 +212,7 @@ func reconcileSharedRootState(ctx context.Context, backend storage.Backend, stat
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func missingSingleOwnerOutputs(ctx context.Context, backend storage.Backend, outputs []state.OutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
if storage.IsNotFound(err) {
|
||||
missing = append(missing, ReconcileStatePath{Path: output.Path, StorageStatus: "missing"})
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outputs []state.SharedRootOutputFile) ([]ReconcileStatePath, error) {
|
||||
func missingCatalogOutputs(ctx context.Context, backend storage.Backend, outputs []state.CatalogOutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
@@ -274,8 +220,8 @@ func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outp
|
||||
missing = append(missing, ReconcileStatePath{
|
||||
Path: output.Path,
|
||||
OwnerScope: &ReconcileStateOwnerScope{
|
||||
PipelineID: output.Owner.PipelineID,
|
||||
DestinationID: output.Owner.DestinationID,
|
||||
PipelineID: output.PipelineID,
|
||||
DestinationID: output.DestinationID,
|
||||
},
|
||||
StorageStatus: "missing",
|
||||
})
|
||||
@@ -323,16 +269,6 @@ func unmanagedEntries(ctx context.Context, backend storage.Backend, managedPaths
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func sharedRootOutputsForOwner(outputs []state.SharedRootOutputFile, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
selected := make([]state.SharedRootOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if output.Owner == scope {
|
||||
selected = append(selected, output)
|
||||
}
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func missingReportPaths(missing []ReconcileStatePath) []string {
|
||||
paths := make([]string, 0, len(missing))
|
||||
for _, item := range missing {
|
||||
|
||||
@@ -2,7 +2,6 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -15,12 +14,11 @@ import (
|
||||
)
|
||||
|
||||
func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testing.T) {
|
||||
t.Skip("catalog reconcile-state execution is covered by the catalog maintenance work")
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
catalog := pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC))
|
||||
writeFakeCatalogState(t, backend, catalog)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed output: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
@@ -36,25 +34,23 @@ func TestReconcileStateDryRunReportsMissingManagedOutputsWithoutRewrite(t *testi
|
||||
if !report.WouldChange || report.Changed {
|
||||
t.Fatalf("report changed=%t would_change=%t, want dry-run pending change", report.Changed, report.WouldChange)
|
||||
}
|
||||
if got := reportPathList(report.MissingManagedOutputs); got != "summary.txt" {
|
||||
t.Fatalf("missing outputs = %q, want summary.txt", got)
|
||||
if got := reportPathList(report.MissingManagedOutputs); got != "fresh.txt" {
|
||||
t.Fatalf("missing outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
if got := entryPathList(report.UnmanagedEntries); got != "extra.txt" {
|
||||
t.Fatalf("unmanaged entries = %q, want extra.txt", got)
|
||||
}
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md,summary.txt" {
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,fresh.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want original outputs", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *testing.T) {
|
||||
t.Skip("catalog reconcile-state execution is covered by the catalog maintenance work")
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
manifest := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
testutil.WriteFakeDestinationState(t, backend, "", manifest, testutil.DestinationStateOptions{})
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"summary.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"fresh.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed output: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
@@ -69,12 +65,15 @@ func TestReconcileStateApplyRemovesMissingRecordsAndPreservesUnmanagedFiles(t *t
|
||||
if !report.Changed || report.WouldChange {
|
||||
t.Fatalf("report changed=%t would_change=%t, want applied change", report.Changed, report.WouldChange)
|
||||
}
|
||||
destinationState := readFakeSingleOwnerState(t, backend)
|
||||
if err := state.Validate(destinationState); err != nil {
|
||||
t.Fatalf("Validate() repaired state error = %v", err)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if err := state.ValidateCatalog(repaired); err != nil {
|
||||
t.Fatalf("ValidateCatalog() repaired state error = %v", err)
|
||||
}
|
||||
if got := strings.Join(state.ManagedOutputPaths(destinationState), ","); got != "report.md" {
|
||||
t.Fatalf("state outputs = %q, want report.md", got)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "old.txt,html.txt" {
|
||||
t.Fatalf("state outputs = %q, want old.txt,html.txt", got)
|
||||
}
|
||||
if repaired.SchemaVersion != state.CatalogSchemaVersion {
|
||||
t.Fatalf("state schema_version = %d, want %d", repaired.SchemaVersion, state.CatalogSchemaVersion)
|
||||
}
|
||||
testutil.AssertFakeFile(t, backend, "extra.txt", "unmanaged")
|
||||
}
|
||||
@@ -101,13 +100,11 @@ func TestReconcileStateInvalidStateFailsWithoutRewrite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
|
||||
t.Skip("catalog reconcile-state execution is covered by the catalog maintenance work")
|
||||
func TestReconcileStateOwnerScopeRepairsCurrentOwnerOnly(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
sharedRoot := reconcileSharedRootFixture(t)
|
||||
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed outputs: %v", err)
|
||||
}
|
||||
|
||||
@@ -121,19 +118,17 @@ func TestReconcileStateSharedRootOwnerScopeRepairsCurrentOwnerOnly(t *testing.T)
|
||||
if !report.Changed {
|
||||
t.Fatal("report changed = false, want true")
|
||||
}
|
||||
repaired := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := strings.Join(repaired.AllManagedOutputPaths(), ","); got != "report.html" {
|
||||
t.Fatalf("shared-root outputs = %q, want other owner output preserved", got)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt,html.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want other owner output preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
t.Skip("catalog reconcile-state execution is covered by the catalog maintenance work")
|
||||
func TestReconcileStateAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
backend := fake.New()
|
||||
cfg := reconcileStateS3Config(t)
|
||||
sharedRoot := reconcileSharedRootFixture(t)
|
||||
writeFakeSharedRootStateForApp(t, backend, sharedRoot)
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"report.md", "report.html"}, storage.DeleteOptions{}); err != nil {
|
||||
writeFakeCatalogState(t, backend, pruneCatalogState(time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)))
|
||||
if err := backend.DeleteManagedOutputs(context.Background(), "", []string{"old.txt", "html.txt"}, storage.DeleteOptions{}); err != nil {
|
||||
t.Fatalf("delete managed outputs: %v", err)
|
||||
}
|
||||
|
||||
@@ -145,12 +140,12 @@ func TestReconcileStateSharedRootAllOwnersRepairsEveryOwner(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileStateConfigWithBackendFactory() error = %v", err)
|
||||
}
|
||||
if !report.Changed || report.CheckedCount != 2 {
|
||||
if !report.Changed || report.CheckedCount != 3 {
|
||||
t.Fatalf("report changed=%t checked=%d, want all-owner repair", report.Changed, report.CheckedCount)
|
||||
}
|
||||
repaired := readFakeSharedRootStateForApp(t, backend)
|
||||
if got := repaired.AllManagedOutputPaths(); len(got) != 0 {
|
||||
t.Fatalf("shared-root outputs = %#v, want none", got)
|
||||
repaired := readFakeCatalogState(t, backend)
|
||||
if got := strings.Join(state.CatalogManagedOutputPaths(repaired), ","); got != "fresh.txt" {
|
||||
t.Fatalf("catalog outputs = %q, want fresh.txt", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,93 +164,6 @@ func reconcileStateS3Config(t *testing.T) config.Config {
|
||||
return cfg
|
||||
}
|
||||
|
||||
func readFakeSingleOwnerState(t *testing.T, backend *fake.Backend) state.DistributorState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read state: %v", err)
|
||||
}
|
||||
destinationState, err := state.Parse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse state: %v", err)
|
||||
}
|
||||
return destinationState
|
||||
}
|
||||
|
||||
func writeFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend, sharedRoot state.SharedRootState) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(sharedRoot, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal shared-root state: %v", err)
|
||||
}
|
||||
testutil.WriteFakeFile(t, backend, storage.StateFileName, string(append(data, '\n')))
|
||||
for _, output := range sharedRoot.Outputs {
|
||||
testutil.WriteFakeFile(t, backend, output.Path, "old")
|
||||
}
|
||||
}
|
||||
|
||||
func readFakeSharedRootStateForApp(t *testing.T, backend *fake.Backend) state.SharedRootState {
|
||||
t.Helper()
|
||||
data, err := backend.ReadFile(context.Background(), storage.StateFileName)
|
||||
if err != nil {
|
||||
t.Fatalf("read shared-root state: %v", err)
|
||||
}
|
||||
sharedRoot, err := state.ParseSharedRoot(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse shared-root state: %v", err)
|
||||
}
|
||||
return sharedRoot
|
||||
}
|
||||
|
||||
func reconcileSharedRootFixture(t *testing.T) state.SharedRootState {
|
||||
t.Helper()
|
||||
source := testutil.ValidManifest(testutil.BundleOptions{})
|
||||
htmlSource := source
|
||||
createdAt := time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC)
|
||||
return state.SharedRootState{
|
||||
SchemaVersion: state.SharedRootSchemaVersion,
|
||||
DistributorVersion: "test",
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
State: state.StatePolicy{Mode: state.StateModeSharedRoot},
|
||||
Owners: []state.OwnerRecord{{
|
||||
Scope: state.CurrentOwnerScope("reports", "archive"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeReplace},
|
||||
Source: state.SourceState{Manifest: source},
|
||||
}, {
|
||||
Scope: state.CurrentOwnerScope("reports", "html"),
|
||||
Reconciliation: state.ReconciliationPolicy{Mode: config.ReconciliationModeMerge},
|
||||
Source: state.SourceState{Manifest: htmlSource},
|
||||
}},
|
||||
Outputs: []state.SharedRootOutputFile{{
|
||||
Path: "report.md",
|
||||
Kind: state.OutputKindSource,
|
||||
SourcePath: "report.md",
|
||||
SHA256: source.Files[0].SHA256,
|
||||
Size: source.Files[0].Size,
|
||||
Owner: state.CurrentOwnerScope("reports", "archive"),
|
||||
SourceID: source.ID,
|
||||
SourceDigest: source.Digest,
|
||||
SourceCreated: source.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}, {
|
||||
Path: "report.html",
|
||||
Kind: state.OutputKindGenerated,
|
||||
SourcePath: "report.md",
|
||||
Transform: "markdown_to_html",
|
||||
SHA256: "sha256:" + strings.Repeat("a", 64),
|
||||
Size: 128,
|
||||
Owner: state.CurrentOwnerScope("reports", "html"),
|
||||
SourceID: htmlSource.ID,
|
||||
SourceDigest: htmlSource.Digest,
|
||||
SourceCreated: htmlSource.Created,
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func reportPathList(paths []ReconcileStatePath) string {
|
||||
values := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
|
||||
Reference in New Issue
Block a user